Various bits; preview video seems to work.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2016 Carl Hetherington <cth@carlh.net>
3
4     This file is part of DCP-o-matic.
5
6     DCP-o-matic is free software; you can redistribute it and/or modify
7     it under the terms of the GNU General Public License as published by
8     the Free Software Foundation; either version 2 of the License, or
9     (at your option) any later version.
10
11     DCP-o-matic is distributed in the hope that it will be useful,
12     but WITHOUT ANY WARRANTY; without even the implied warranty of
13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14     GNU General Public License for more details.
15
16     You should have received a copy of the GNU General Public License
17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
18
19 */
20
21 #include "player.h"
22 #include "film.h"
23 #include "audio_buffers.h"
24 #include "content_audio.h"
25 #include "dcp_content.h"
26 #include "job.h"
27 #include "image.h"
28 #include "raw_image_proxy.h"
29 #include "ratio.h"
30 #include "log.h"
31 #include "render_subtitles.h"
32 #include "config.h"
33 #include "content_video.h"
34 #include "player_video.h"
35 #include "frame_rate_change.h"
36 #include "audio_processor.h"
37 #include "playlist.h"
38 #include "referenced_reel_asset.h"
39 #include "decoder_factory.h"
40 #include "decoder.h"
41 #include "video_decoder.h"
42 #include "audio_decoder.h"
43 #include "subtitle_content.h"
44 #include "subtitle_decoder.h"
45 #include "ffmpeg_content.h"
46 #include "audio_content.h"
47 #include "content_subtitle.h"
48 #include "dcp_decoder.h"
49 #include "image_decoder.h"
50 #include <dcp/reel.h>
51 #include <dcp/reel_sound_asset.h>
52 #include <dcp/reel_subtitle_asset.h>
53 #include <dcp/reel_picture_asset.h>
54 #include <boost/foreach.hpp>
55 #include <stdint.h>
56 #include <algorithm>
57 #include <iostream>
58
59 #include "i18n.h"
60
61 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
62
63 using std::list;
64 using std::cout;
65 using std::min;
66 using std::max;
67 using std::min;
68 using std::vector;
69 using std::pair;
70 using std::map;
71 using std::make_pair;
72 using std::copy;
73 using boost::shared_ptr;
74 using boost::weak_ptr;
75 using boost::dynamic_pointer_cast;
76 using boost::optional;
77 using boost::scoped_ptr;
78
79 static bool
80 has_video (Content* c)
81 {
82         return static_cast<bool>(c->video);
83 }
84
85 static bool
86 has_audio (Content* c)
87 {
88         return static_cast<bool>(c->audio);
89 }
90
91 static bool
92 has_subtitle (Content* c)
93 {
94         return static_cast<bool>(c->subtitle);
95 }
96
97 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
98         : _film (film)
99         , _playlist (playlist)
100         , _have_valid_pieces (false)
101         , _ignore_video (false)
102         , _ignore_audio (false)
103         , _always_burn_subtitles (false)
104         , _fast (false)
105         , _play_referenced (false)
106         , _audio_merger (_film->audio_channels(), _film->audio_frame_rate())
107 {
108         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
109         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
110         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
111         set_video_container_size (_film->frame_size ());
112
113         film_changed (Film::AUDIO_PROCESSOR);
114 }
115
116 void
117 Player::setup_pieces ()
118 {
119         _pieces.clear ();
120
121         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
122
123                 if (!i->paths_valid ()) {
124                         continue;
125                 }
126
127                 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log());
128                 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
129
130                 if (!decoder) {
131                         /* Not something that we can decode; e.g. Atmos content */
132                         continue;
133                 }
134
135                 if (decoder->video && _ignore_video) {
136                         decoder->video->set_ignore ();
137                 }
138
139                 if (decoder->audio && _ignore_audio) {
140                         decoder->audio->set_ignore ();
141                 }
142
143                 if (decoder->audio && _fast) {
144                         decoder->audio->set_fast ();
145                 }
146
147                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
148                 if (dcp && _play_referenced) {
149                         dcp->set_decode_referenced ();
150                 }
151
152                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
153                 _pieces.push_back (piece);
154
155                 if (decoder->video) {
156                         decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece> (piece), _1));
157                 }
158
159                 if (decoder->audio) {
160                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
161                 }
162
163                 if (decoder->subtitle) {
164                         decoder->subtitle->ImageData.connect (bind (&Player::image_subtitle, this, weak_ptr<Piece> (piece), _1));
165                         decoder->subtitle->TextData.connect (bind (&Player::text_subtitle, this, weak_ptr<Piece> (piece), _1));
166                 }
167         }
168
169         _have_valid_pieces = true;
170 }
171
172 void
173 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
174 {
175         shared_ptr<Content> c = w.lock ();
176         if (!c) {
177                 return;
178         }
179
180         if (
181                 property == ContentProperty::POSITION ||
182                 property == ContentProperty::LENGTH ||
183                 property == ContentProperty::TRIM_START ||
184                 property == ContentProperty::TRIM_END ||
185                 property == ContentProperty::PATH ||
186                 property == VideoContentProperty::FRAME_TYPE ||
187                 property == DCPContentProperty::NEEDS_ASSETS ||
188                 property == DCPContentProperty::NEEDS_KDM ||
189                 property == SubtitleContentProperty::COLOUR ||
190                 property == SubtitleContentProperty::OUTLINE ||
191                 property == SubtitleContentProperty::SHADOW ||
192                 property == SubtitleContentProperty::EFFECT_COLOUR ||
193                 property == FFmpegContentProperty::SUBTITLE_STREAM ||
194                 property == VideoContentProperty::COLOUR_CONVERSION
195                 ) {
196
197                 _have_valid_pieces = false;
198                 Changed (frequent);
199
200         } else if (
201                 property == SubtitleContentProperty::LINE_SPACING ||
202                 property == SubtitleContentProperty::OUTLINE_WIDTH ||
203                 property == SubtitleContentProperty::Y_SCALE ||
204                 property == SubtitleContentProperty::FADE_IN ||
205                 property == SubtitleContentProperty::FADE_OUT ||
206                 property == ContentProperty::VIDEO_FRAME_RATE ||
207                 property == SubtitleContentProperty::USE ||
208                 property == SubtitleContentProperty::X_OFFSET ||
209                 property == SubtitleContentProperty::Y_OFFSET ||
210                 property == SubtitleContentProperty::X_SCALE ||
211                 property == SubtitleContentProperty::FONTS ||
212                 property == VideoContentProperty::CROP ||
213                 property == VideoContentProperty::SCALE ||
214                 property == VideoContentProperty::FADE_IN ||
215                 property == VideoContentProperty::FADE_OUT
216                 ) {
217
218                 Changed (frequent);
219         }
220 }
221
222 void
223 Player::set_video_container_size (dcp::Size s)
224 {
225         _video_container_size = s;
226
227         _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
228         _black_image->make_black ();
229 }
230
231 void
232 Player::playlist_changed ()
233 {
234         _have_valid_pieces = false;
235         Changed (false);
236 }
237
238 void
239 Player::film_changed (Film::Property p)
240 {
241         /* Here we should notice Film properties that affect our output, and
242            alert listeners that our output now would be different to how it was
243            last time we were run.
244         */
245
246         if (p == Film::CONTAINER) {
247                 Changed (false);
248         } else if (p == Film::VIDEO_FRAME_RATE) {
249                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
250                    so we need new pieces here.
251                 */
252                 _have_valid_pieces = false;
253                 Changed (false);
254         } else if (p == Film::AUDIO_PROCESSOR) {
255                 if (_film->audio_processor ()) {
256                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
257                 }
258         }
259 }
260
261 list<PositionImage>
262 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
263 {
264         list<PositionImage> all;
265
266         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
267                 if (!i->image) {
268                         continue;
269                 }
270
271                 /* We will scale the subtitle up to fit _video_container_size */
272                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
273
274                 /* Then we need a corrective translation, consisting of two parts:
275                  *
276                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
277                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
278                  *
279                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
280                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
281                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
282                  *
283                  * Combining these two translations gives these expressions.
284                  */
285
286                 all.push_back (
287                         PositionImage (
288                                 i->image->scale (
289                                         scaled_size,
290                                         dcp::YUV_TO_RGB_REC601,
291                                         i->image->pixel_format (),
292                                         true,
293                                         _fast
294                                         ),
295                                 Position<int> (
296                                         lrint (_video_container_size.width * i->rectangle.x),
297                                         lrint (_video_container_size.height * i->rectangle.y)
298                                         )
299                                 )
300                         );
301         }
302
303         return all;
304 }
305
306 shared_ptr<PlayerVideo>
307 Player::black_player_video_frame (DCPTime time) const
308 {
309         return shared_ptr<PlayerVideo> (
310                 new PlayerVideo (
311                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
312                         time,
313                         Crop (),
314                         optional<double> (),
315                         _video_container_size,
316                         _video_container_size,
317                         EYES_BOTH,
318                         PART_WHOLE,
319                         PresetColourConversion::all().front().conversion
320                 )
321         );
322 }
323
324 Frame
325 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
326 {
327         DCPTime s = t - piece->content->position ();
328         s = min (piece->content->length_after_trim(), s);
329         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
330
331         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
332            then convert that ContentTime to frames at the content's rate.  However this fails for
333            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
334            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
335
336            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
337         */
338         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
339 }
340
341 DCPTime
342 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
343 {
344         /* See comment in dcp_to_content_video */
345         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime (piece->content->trim_start (), piece->frc);
346         return max (DCPTime (), d + piece->content->position ());
347 }
348
349 Frame
350 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
351 {
352         DCPTime s = t - piece->content->position ();
353         s = min (piece->content->length_after_trim(), s);
354         /* See notes in dcp_to_content_video */
355         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
356 }
357
358 DCPTime
359 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
360 {
361         /* See comment in dcp_to_content_video */
362         DCPTime const d = DCPTime::from_frames (f, _film->audio_frame_rate()) - DCPTime (piece->content->trim_start (), piece->frc);
363         return max (DCPTime (), d + piece->content->position ());
364 }
365
366 ContentTime
367 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
368 {
369         DCPTime s = t - piece->content->position ();
370         s = min (piece->content->length_after_trim(), s);
371         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
372 }
373
374 DCPTime
375 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
376 {
377         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
378 }
379
380 list<shared_ptr<Font> >
381 Player::get_subtitle_fonts ()
382 {
383         if (!_have_valid_pieces) {
384                 setup_pieces ();
385         }
386
387         list<shared_ptr<Font> > fonts;
388         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
389                 if (p->content->subtitle) {
390                         /* XXX: things may go wrong if there are duplicate font IDs
391                            with different font files.
392                         */
393                         list<shared_ptr<Font> > f = p->content->subtitle->fonts ();
394                         copy (f.begin(), f.end(), back_inserter (fonts));
395                 }
396         }
397
398         return fonts;
399 }
400
401 /** Set this player never to produce any video data */
402 void
403 Player::set_ignore_video ()
404 {
405         _ignore_video = true;
406 }
407
408 /** Set this player never to produce any audio data */
409 void
410 Player::set_ignore_audio ()
411 {
412         _ignore_audio = true;
413 }
414
415 /** Set whether or not this player should always burn text subtitles into the image,
416  *  regardless of the content settings.
417  *  @param burn true to always burn subtitles, false to obey content settings.
418  */
419 void
420 Player::set_always_burn_subtitles (bool burn)
421 {
422         _always_burn_subtitles = burn;
423 }
424
425 void
426 Player::set_fast ()
427 {
428         _fast = true;
429         _have_valid_pieces = false;
430 }
431
432 void
433 Player::set_play_referenced ()
434 {
435         _play_referenced = true;
436         _have_valid_pieces = false;
437 }
438
439 list<ReferencedReelAsset>
440 Player::get_reel_assets ()
441 {
442         list<ReferencedReelAsset> a;
443
444         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
445                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
446                 if (!j) {
447                         continue;
448                 }
449
450                 scoped_ptr<DCPDecoder> decoder;
451                 try {
452                         decoder.reset (new DCPDecoder (j, _film->log()));
453                 } catch (...) {
454                         return a;
455                 }
456
457                 int64_t offset = 0;
458                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
459
460                         DCPOMATIC_ASSERT (j->video_frame_rate ());
461                         double const cfr = j->video_frame_rate().get();
462                         Frame const trim_start = j->trim_start().frames_round (cfr);
463                         Frame const trim_end = j->trim_end().frames_round (cfr);
464                         int const ffr = _film->video_frame_rate ();
465
466                         DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
467                         if (j->reference_video ()) {
468                                 shared_ptr<dcp::ReelAsset> ra = k->main_picture ();
469                                 DCPOMATIC_ASSERT (ra);
470                                 ra->set_entry_point (ra->entry_point() + trim_start);
471                                 ra->set_duration (ra->duration() - trim_start - trim_end);
472                                 a.push_back (
473                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
474                                         );
475                         }
476
477                         if (j->reference_audio ()) {
478                                 shared_ptr<dcp::ReelAsset> ra = k->main_sound ();
479                                 DCPOMATIC_ASSERT (ra);
480                                 ra->set_entry_point (ra->entry_point() + trim_start);
481                                 ra->set_duration (ra->duration() - trim_start - trim_end);
482                                 a.push_back (
483                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
484                                         );
485                         }
486
487                         if (j->reference_subtitle ()) {
488                                 shared_ptr<dcp::ReelAsset> ra = k->main_subtitle ();
489                                 DCPOMATIC_ASSERT (ra);
490                                 ra->set_entry_point (ra->entry_point() + trim_start);
491                                 ra->set_duration (ra->duration() - trim_start - trim_end);
492                                 a.push_back (
493                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
494                                         );
495                         }
496
497                         /* Assume that main picture duration is the length of the reel */
498                         offset += k->main_picture()->duration ();
499                 }
500         }
501
502         return a;
503 }
504
505 list<shared_ptr<Piece> >
506 Player::overlaps (DCPTime from, DCPTime to, boost::function<bool (Content *)> valid)
507 {
508         if (!_have_valid_pieces) {
509                 setup_pieces ();
510         }
511
512         list<shared_ptr<Piece> > overlaps;
513         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
514                 if (valid (i->content.get ()) && i->content->position() < to && i->content->end() > from) {
515                         overlaps.push_back (i);
516                 }
517         }
518
519         return overlaps;
520 }
521
522 bool
523 Player::pass ()
524 {
525         if (!_have_valid_pieces) {
526                 setup_pieces ();
527         }
528
529         shared_ptr<Piece> earliest;
530         DCPTime earliest_position;
531         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
532                 DCPTime const t = i->content->position() + DCPTime (i->decoder->position(), i->frc);
533                 if (!earliest || t < earliest_position) {
534                         earliest_position = t;
535                         earliest = i;
536                 }
537         }
538
539         if (!earliest) {
540                 return true;
541         }
542
543         cout << "Pass " << earliest->content->path(0) << "\n";
544         earliest->decoder->pass ();
545
546         /* Emit any audio that is ready */
547
548         pair<shared_ptr<AudioBuffers>, DCPTime> audio = _audio_merger.pull (earliest_position);
549         if (audio.first->frames() > 0) {
550                 DCPOMATIC_ASSERT (audio.second >= _last_audio_time);
551                 DCPTime t = _last_audio_time;
552                 while (t < audio.second) {
553                         /* Silence up to the time of this new audio */
554                         DCPTime block = min (DCPTime::from_seconds (0.5), audio.second - t);
555                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), block.frames_round(_film->audio_frame_rate())));
556                         silence->make_silent ();
557                         Audio (silence, t);
558                         t += block;
559                 }
560
561                 Audio (audio.first, audio.second);
562                 _last_audio_time = audio.second;
563         }
564
565         return false;
566 }
567
568 void
569 Player::video (weak_ptr<Piece> wp, ContentVideo video)
570 {
571         shared_ptr<Piece> piece = wp.lock ();
572         if (!piece) {
573                 return;
574         }
575
576         /* XXX: get subs to burn in and burn them in */
577
578
579         DCPTime const time = content_video_to_dcp (piece, video.frame.index());
580
581         /* Fill gaps */
582
583         if (_last_video_time) {
584                 for (DCPTime i = _last_video_time.get(); i < time; i += DCPTime::from_frames (1, _film->video_frame_rate())) {
585                         if (_playlist->video_content_at(i) && _last_video) {
586                                 Video (_last_video->clone (i));
587                         } else {
588                                 Video (black_player_video_frame (i));
589                         }
590                 }
591         }
592
593         _last_video.reset (
594                 new PlayerVideo (
595                         video.image,
596                         time,
597                         piece->content->video->crop (),
598                         piece->content->video->fade (video.frame.index()),
599                         piece->content->video->scale().size (
600                                 piece->content->video, _video_container_size, _film->frame_size ()
601                                 ),
602                         _video_container_size,
603                         video.frame.eyes(),
604                         video.part,
605                         piece->content->video->colour_conversion ()
606                         )
607                 );
608
609         _last_video_time = time;
610
611         cout << "Video @ " << to_string(_last_video_time.get()) << "\n";
612         Video (_last_video);
613 }
614
615 void
616 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
617 {
618         shared_ptr<Piece> piece = wp.lock ();
619         if (!piece) {
620                 return;
621         }
622
623         shared_ptr<AudioContent> content = piece->content->audio;
624         DCPOMATIC_ASSERT (content);
625
626         shared_ptr<AudioBuffers> audio = content_audio.audio;
627
628         /* Gain */
629         if (content->gain() != 0) {
630                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
631                 gain->apply_gain (content->gain ());
632                 audio = gain;
633         }
634
635         /* XXX: end-trimming used to be checked here */
636
637         /* Compute time in the DCP */
638         DCPTime const time = resampled_audio_to_dcp (piece, content_audio.frame) + DCPTime::from_seconds (content->delay() / 1000);
639
640         /* Remap channels */
641         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
642         dcp_mapped->make_silent ();
643
644         AudioMapping map = stream->mapping ();
645         for (int i = 0; i < map.input_channels(); ++i) {
646                 for (int j = 0; j < dcp_mapped->channels(); ++j) {
647                         if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
648                                 dcp_mapped->accumulate_channel (
649                                         audio.get(),
650                                         i,
651                                         static_cast<dcp::Channel> (j),
652                                         map.get (i, static_cast<dcp::Channel> (j))
653                                         );
654                         }
655                 }
656         }
657
658         audio = dcp_mapped;
659
660         if (_audio_processor) {
661                 audio = _audio_processor->run (audio, _film->audio_channels ());
662         }
663
664         _audio_merger.push (audio, time);
665 }
666
667 void
668 Player::image_subtitle (weak_ptr<Piece> piece, ContentImageSubtitle subtitle)
669 {
670         /* XXX: Store for video to see */
671 }
672
673 void
674 Player::text_subtitle (weak_ptr<Piece> piece, ContentTextSubtitle subtitle)
675 {
676         /* XXX: Store for video to see, or emit */
677 }
678
679 void
680 Player::seek (DCPTime time, bool accurate)
681 {
682         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
683                 if (i->content->position() <= time && time < i->content->end()) {
684                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
685                 }
686         }
687
688         if (accurate) {
689                 _last_video_time = time - DCPTime::from_frames (1, _film->video_frame_rate ());
690         } else {
691                 _last_video_time = optional<DCPTime> ();
692         }
693 }