96d23a82b38c3244cec08e01f68016d43f52f60c
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include <algorithm>
22 #include "player.h"
23 #include "film.h"
24 #include "ffmpeg_decoder.h"
25 #include "ffmpeg_content.h"
26 #include "image_decoder.h"
27 #include "image_content.h"
28 #include "sndfile_decoder.h"
29 #include "sndfile_content.h"
30 #include "subtitle_content.h"
31 #include "playlist.h"
32 #include "job.h"
33 #include "image.h"
34 #include "ratio.h"
35 #include "log.h"
36 #include "scaler.h"
37
38 using std::list;
39 using std::cout;
40 using std::min;
41 using std::max;
42 using std::vector;
43 using std::pair;
44 using std::map;
45 using boost::shared_ptr;
46 using boost::weak_ptr;
47 using boost::dynamic_pointer_cast;
48 using boost::optional;
49
50 class Piece
51 {
52 public:
53         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
54                 : content (c)
55                 , decoder (d)
56                 , frc (f)
57         {}
58
59         shared_ptr<Content> content;
60         shared_ptr<Decoder> decoder;
61         FrameRateChange frc;
62 };
63
64 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
65         : _film (f)
66         , _playlist (p)
67         , _video (true)
68         , _audio (true)
69         , _have_valid_pieces (false)
70         , _video_position (0)
71         , _audio_position (0)
72         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
73         , _last_emit_was_black (false)
74         , _just_did_inaccurate_seek (false)
75         , _approximate_size (false)
76 {
77         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
78         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
79         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
80         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
81 }
82
83 void
84 Player::disable_video ()
85 {
86         _video = false;
87 }
88
89 void
90 Player::disable_audio ()
91 {
92         _audio = false;
93 }
94
95 bool
96 Player::pass ()
97 {
98         if (!_have_valid_pieces) {
99                 setup_pieces ();
100         }
101
102         /* Interrogate all our pieces to find the one with the earliest decoded data */
103
104         shared_ptr<Piece> earliest_piece;
105         shared_ptr<Decoded> earliest_decoded;
106         DCPTime earliest_time = TIME_MAX;
107         DCPTime earliest_audio = TIME_MAX;
108
109         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
110
111                 shared_ptr<Decoded> dec = (*i)->decoder->peek ();
112
113                 if (dec) {
114                         dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position() - (*i)->content->trim_start());
115                 }
116
117                 if (dec && dec->dcp_time < earliest_time) {
118                         earliest_piece = *i;
119                         earliest_decoded = dec;
120                         earliest_time = dec->dcp_time;
121                 }
122
123                 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
124                         earliest_audio = dec->dcp_time;
125                 }
126         }
127                 
128         if (!earliest_piece) {
129                 flush ();
130                 return true;
131         }
132
133         if (earliest_audio != TIME_MAX) {
134                 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (max (int64_t (0), earliest_audio));
135                 Audio (tb.audio, tb.time);
136                 /* This assumes that the audio_frames_to_time conversion is exact
137                    so that there are no accumulated errors caused by rounding.
138                 */
139                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
140         }
141
142         /* Emit the earliest thing */
143
144         shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
145         shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
146         shared_ptr<DecodedSubtitle> ds = dynamic_pointer_cast<DecodedSubtitle> (earliest_decoded);
147
148         /* Will be set to false if we shouldn't consume the peeked DecodedThing */
149         bool consume = true;
150
151         /* This is the margin either side of _{video,audio}_position that we will accept
152            as a starting point for a frame consecutive to the previous.
153         */
154         DCPTime const margin = TIME_HZ / (2 * _film->video_frame_rate ());
155         
156         if (dv && _video) {
157
158                 if (_just_did_inaccurate_seek) {
159
160                         /* Just emit; no subtlety */
161                         emit_video (earliest_piece, dv);
162                         step_video_position (dv);
163                         
164                 } else if (dv->dcp_time - _video_position > margin) {
165
166                         /* Too far ahead */
167
168                         list<shared_ptr<Piece> >::iterator i = _pieces.begin();
169                         while (i != _pieces.end() && ((*i)->content->position() >= _video_position || _video_position >= (*i)->content->end())) {
170                                 ++i;
171                         }
172
173                         if (i == _pieces.end() || !_last_incoming_video.video || !_have_valid_pieces) {
174                                 /* We're outside all video content */
175                                 emit_black ();
176                         } else {
177                                 /* We're inside some video; repeat the frame */
178                                 _last_incoming_video.video->dcp_time = _video_position;
179                                 emit_video (_last_incoming_video.weak_piece, _last_incoming_video.video);
180                                 step_video_position (_last_incoming_video.video);
181                         }
182
183                         consume = false;
184
185                 } else if (abs (dv->dcp_time - _video_position) < margin) {
186                         /* We're ok */
187                         emit_video (earliest_piece, dv);
188                         step_video_position (dv);
189                 } else {
190                         /* Too far behind: skip */
191                 }
192
193                 _just_did_inaccurate_seek = false;
194
195         } else if (da && _audio) {
196
197                 if (da->dcp_time - _audio_position > margin) {
198                         /* Too far ahead */
199                         emit_silence (da->dcp_time - _audio_position);
200                         consume = false;
201                 } else if (abs (da->dcp_time - _audio_position) < margin) {
202                         /* We're ok */
203                         emit_audio (earliest_piece, da);
204                 } else {
205                         /* Too far behind: skip */
206                 }
207                 
208         } else if (ds && _video) {
209                 _in_subtitle.piece = earliest_piece;
210                 _in_subtitle.subtitle = ds;
211                 update_subtitle ();
212         }
213
214         if (consume) {
215                 earliest_piece->decoder->consume ();
216         }                       
217         
218         return false;
219 }
220
221 void
222 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
223 {
224         /* Keep a note of what came in so that we can repeat it if required */
225         _last_incoming_video.weak_piece = weak_piece;
226         _last_incoming_video.video = video;
227         
228         shared_ptr<Piece> piece = weak_piece.lock ();
229         if (!piece) {
230                 return;
231         }
232
233         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
234         assert (content);
235
236         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
237
238         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
239         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
240         if (_approximate_size) {
241                 image_size.width &= ~3;
242                 image_size.height &= ~3;
243         }
244
245         shared_ptr<PlayerImage> pi (
246                 new PlayerImage (
247                         video->image,
248                         content->crop(),
249                         image_size,
250                         _video_container_size,
251                         _film->scaler()
252                         )
253                 );
254         
255         if (
256                 _film->with_subtitles () &&
257                 _out_subtitle.image &&
258                 video->dcp_time >= _out_subtitle.from && video->dcp_time <= _out_subtitle.to
259                 ) {
260
261                 Position<int> const container_offset (
262                         (_video_container_size.width - image_size.width) / 2,
263                         (_video_container_size.height - image_size.height) / 2
264                         );
265
266                 pi->set_subtitle (_out_subtitle.image, _out_subtitle.position + container_offset);
267         }
268                                             
269 #ifdef DCPOMATIC_DEBUG
270         _last_video = piece->content;
271 #endif
272
273         Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
274         
275         _last_emit_was_black = false;
276 }
277
278 void
279 Player::step_video_position (shared_ptr<DecodedVideo> video)
280 {
281         /* This is a bit of a hack; don't update _video_position if EYES_RIGHT is on its way */
282         if (video->eyes != EYES_LEFT) {
283                 /* This assumes that the video_frames_to_time conversion is exact
284                    so that there are no accumulated errors caused by rounding.
285                 */
286                 _video_position += _film->video_frames_to_time (1);
287         }
288 }
289
290 void
291 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
292 {
293         shared_ptr<Piece> piece = weak_piece.lock ();
294         if (!piece) {
295                 return;
296         }
297
298         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
299         assert (content);
300
301         /* Gain */
302         if (content->audio_gain() != 0) {
303                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
304                 gain->apply_gain (content->audio_gain ());
305                 audio->data = gain;
306         }
307
308         if (content->trimmed (audio->dcp_time - content->position ())) {
309                 return;
310         }
311
312         /* Remap channels */
313         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
314         dcp_mapped->make_silent ();
315         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
316         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
317                 if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) {
318                         dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second);
319                 }
320         }
321
322         audio->data = dcp_mapped;
323
324         /* Delay */
325         audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
326         if (audio->dcp_time < 0) {
327                 int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
328                 if (frames >= audio->data->frames ()) {
329                         return;
330                 }
331
332                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
333                 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
334
335                 audio->data = trimmed;
336                 audio->dcp_time = 0;
337         }
338
339         _audio_merger.push (audio->data, audio->dcp_time);
340 }
341
342 void
343 Player::flush ()
344 {
345         TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
346         if (tb.audio) {
347                 Audio (tb.audio, tb.time);
348                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
349         }
350
351         while (_video_position < _audio_position) {
352                 emit_black ();
353         }
354
355         while (_audio_position < _video_position) {
356                 emit_silence (_video_position - _audio_position);
357         }
358         
359 }
360
361 /** Seek so that the next pass() will yield (approximately) the requested frame.
362  *  Pass accurate = true to try harder to get close to the request.
363  *  @return true on error
364  */
365 void
366 Player::seek (DCPTime t, bool accurate)
367 {
368         if (!_have_valid_pieces) {
369                 setup_pieces ();
370         }
371
372         if (_pieces.empty ()) {
373                 return;
374         }
375
376         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
377                 /* s is the offset of t from the start position of this content */
378                 DCPTime s = t - (*i)->content->position ();
379                 s = max (static_cast<DCPTime> (0), s);
380                 s = min ((*i)->content->length_after_trim(), s);
381
382                 /* Convert this to the content time */
383                 ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start ();
384
385                 /* And seek the decoder */
386                 (*i)->decoder->seek (ct, accurate);
387         }
388
389         _video_position = time_round_up (t, TIME_HZ / _film->video_frame_rate());
390         _audio_position = time_round_up (t, TIME_HZ / _film->audio_frame_rate());
391
392         _audio_merger.clear (_audio_position);
393
394         if (!accurate) {
395                 /* We just did an inaccurate seek, so it's likely that the next thing seen
396                    out of pass() will be a fair distance from _{video,audio}_position.  Setting
397                    this flag stops pass() from trying to fix that: we assume that if it
398                    was an inaccurate seek then the caller does not care too much about
399                    inserting black/silence to keep the time tidy.
400                 */
401                 _just_did_inaccurate_seek = true;
402         }
403 }
404
405 void
406 Player::setup_pieces ()
407 {
408         list<shared_ptr<Piece> > old_pieces = _pieces;
409         _pieces.clear ();
410
411         ContentList content = _playlist->content ();
412
413         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
414
415                 shared_ptr<Decoder> decoder;
416                 optional<FrameRateChange> frc;
417
418                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
419                 if (fc) {
420                         decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
421                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
422                 }
423                 
424                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
425                 if (ic) {
426                         /* See if we can re-use an old ImageDecoder */
427                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
428                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
429                                 if (imd && imd->content() == ic) {
430                                         decoder = imd;
431                                 }
432                         }
433
434                         if (!decoder) {
435                                 decoder.reset (new ImageDecoder (_film, ic));
436                         }
437
438                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
439                 }
440
441                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
442                 if (sc) {
443                         decoder.reset (new SndfileDecoder (_film, sc));
444
445                         /* Working out the frc for this content is a bit tricky: what if it overlaps
446                            two pieces of video content with different frame rates?  For now, use
447                            the one with the best overlap.
448                         */
449
450                         DCPTime best_overlap_t = 0;
451                         shared_ptr<VideoContent> best_overlap;
452                         for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
453                                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
454                                 if (!vc) {
455                                         continue;
456                                 }
457
458                                 DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end());
459                                 if (overlap > best_overlap_t) {
460                                         best_overlap = vc;
461                                         best_overlap_t = overlap;
462                                 }
463                         }
464
465                         if (best_overlap) {
466                                 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
467                         } else {
468                                 /* No video overlap; e.g. if the DCP is just audio */
469                                 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
470                         }
471                 }
472
473                 decoder->seek ((*i)->trim_start (), true);
474                 
475                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
476         }
477
478         _have_valid_pieces = true;
479
480         /* The Piece for the _last_incoming_video will no longer be valid */
481         _last_incoming_video.video.reset ();
482
483         _video_position = _audio_position = 0;
484 }
485
486 void
487 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
488 {
489         shared_ptr<Content> c = w.lock ();
490         if (!c) {
491                 return;
492         }
493
494         if (
495                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
496                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
497                 property == VideoContentProperty::VIDEO_FRAME_TYPE 
498                 ) {
499                 
500                 _have_valid_pieces = false;
501                 Changed (frequent);
502
503         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
504
505                 update_subtitle ();
506                 Changed (frequent);
507
508         } else if (property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO) {
509                 
510                 Changed (frequent);
511
512         } else if (property == ContentProperty::PATH) {
513
514                 Changed (frequent);
515         }
516 }
517
518 void
519 Player::playlist_changed ()
520 {
521         _have_valid_pieces = false;
522         Changed (false);
523 }
524
525 void
526 Player::set_video_container_size (libdcp::Size s)
527 {
528         _video_container_size = s;
529
530         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
531         im->make_black ();
532         
533         _black_frame.reset (
534                 new PlayerImage (
535                         im,
536                         Crop(),
537                         _video_container_size,
538                         _video_container_size,
539                         Scaler::from_id ("bicubic")
540                         )
541                 );
542 }
543
544 void
545 Player::emit_black ()
546 {
547 #ifdef DCPOMATIC_DEBUG
548         _last_video.reset ();
549 #endif
550
551         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
552         _video_position += _film->video_frames_to_time (1);
553         _last_emit_was_black = true;
554 }
555
556 void
557 Player::emit_silence (DCPTime most)
558 {
559         if (most == 0) {
560                 return;
561         }
562         
563         DCPTime t = min (most, TIME_HZ / 2);
564         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ));
565         silence->make_silent ();
566         Audio (silence, _audio_position);
567         
568         _audio_position += t;
569 }
570
571 void
572 Player::film_changed (Film::Property p)
573 {
574         /* Here we should notice Film properties that affect our output, and
575            alert listeners that our output now would be different to how it was
576            last time we were run.
577         */
578
579         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
580                 Changed (false);
581         }
582 }
583
584 void
585 Player::update_subtitle ()
586 {
587         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
588         if (!piece) {
589                 return;
590         }
591
592         if (!_in_subtitle.subtitle->image) {
593                 _out_subtitle.image.reset ();
594                 return;
595         }
596
597         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
598         assert (sc);
599
600         dcpomatic::Rect<double> in_rect = _in_subtitle.subtitle->rect;
601         libdcp::Size scaled_size;
602
603         in_rect.y += sc->subtitle_offset ();
604
605         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
606         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
607         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
608
609         /* Then we need a corrective translation, consisting of two parts:
610          *
611          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
612          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
613          *
614          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
615          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
616          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
617          *
618          * Combining these two translations gives these expressions.
619          */
620         
621         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
622         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
623
624         _out_subtitle.image = _in_subtitle.subtitle->image->scale (
625                 scaled_size,
626                 Scaler::from_id ("bicubic"),
627                 PIX_FMT_RGBA,
628                 true
629                 );
630
631         _out_subtitle.from = _in_subtitle.subtitle->dcp_time;
632         _out_subtitle.to = _in_subtitle.subtitle->dcp_time_to;
633 }
634
635 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
636  *  @return false if this could not be done.
637  */
638 bool
639 Player::repeat_last_video ()
640 {
641         if (!_last_incoming_video.video || !_have_valid_pieces) {
642                 return false;
643         }
644
645         emit_video (
646                 _last_incoming_video.weak_piece,
647                 _last_incoming_video.video
648                 );
649
650         return true;
651 }
652
653 void
654 Player::set_approximate_size ()
655 {
656         _approximate_size = true;
657 }
658                               
659
660 PlayerImage::PlayerImage (
661         shared_ptr<const Image> in,
662         Crop crop,
663         libdcp::Size inter_size,
664         libdcp::Size out_size,
665         Scaler const * scaler
666         )
667         : _in (in)
668         , _crop (crop)
669         , _inter_size (inter_size)
670         , _out_size (out_size)
671         , _scaler (scaler)
672 {
673
674 }
675
676 void
677 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
678 {
679         _subtitle_image = image;
680         _subtitle_position = pos;
681 }
682
683 shared_ptr<Image>
684 PlayerImage::image (AVPixelFormat format, bool aligned)
685 {
686         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, format, aligned);
687         
688         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
689
690         if (_subtitle_image) {
691                 out->alpha_blend (_subtitle_image, _subtitle_position);
692         }
693
694         return out;
695 }
696