Considerable rework of decoder timing; tests pass, at least.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "playlist.h"
31 #include "job.h"
32 #include "image.h"
33 #include "ratio.h"
34 #include "resampler.h"
35 #include "log.h"
36 #include "scaler.h"
37
38 using std::list;
39 using std::cout;
40 using std::min;
41 using std::max;
42 using std::vector;
43 using std::pair;
44 using std::map;
45 using boost::shared_ptr;
46 using boost::weak_ptr;
47 using boost::dynamic_pointer_cast;
48 using boost::optional;
49
50 class Piece
51 {
52 public:
53         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
54                 : content (c)
55                 , decoder (d)
56                 , frc (f)
57         {}
58
59         shared_ptr<Content> content;
60         shared_ptr<Decoder> decoder;
61         FrameRateChange frc;
62 };
63
64 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
65         : _film (f)
66         , _playlist (p)
67         , _video (true)
68         , _audio (true)
69         , _have_valid_pieces (false)
70         , _video_position (0)
71         , _audio_position (0)
72         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
73         , _last_emit_was_black (false)
74         , _just_did_inaccurate_seek (false)
75 {
76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
79         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
80 }
81
82 void
83 Player::disable_video ()
84 {
85         _video = false;
86 }
87
88 void
89 Player::disable_audio ()
90 {
91         _audio = false;
92 }
93
94 bool
95 Player::pass ()
96 {
97         if (!_have_valid_pieces) {
98                 setup_pieces ();
99         }
100
101         /* Interrogate all our pieces to find the one with the earliest decoded data */
102
103         shared_ptr<Piece> earliest_piece;
104         shared_ptr<Decoded> earliest_decoded;
105         DCPTime earliest_time = TIME_MAX;
106         DCPTime earliest_audio = TIME_MAX;
107
108         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
109
110                 shared_ptr<Decoded> dec = (*i)->decoder->peek ();
111
112                 if (dec) {
113                         dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position());
114                 }
115
116                 /* XXX: don't know what to do with this */
117 #if 0           
118                 if (ad->done()) {
119                         shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> ((*i)->content);
120                         assert (ac);
121                         shared_ptr<Resampler> re = resampler (ac, false);
122                         if (re) {
123                                 shared_ptr<const AudioBuffers> b = re->flush ();
124                                 if (b->frames ()) {
125                                         process_audio (earliest, b, ac->audio_length ());
126                                 }
127                         }
128                 }
129 #endif          
130
131                 if (dec && dec->dcp_time < earliest_time) {
132                         earliest_piece = *i;
133                         earliest_decoded = dec;
134                         earliest_time = dec->dcp_time;
135                 }
136
137                 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
138                         earliest_audio = dec->dcp_time;
139                 }
140         }
141                 
142         if (!earliest_piece) {
143                 flush ();
144                 return true;
145         }
146
147         if (earliest_audio != TIME_MAX) {
148                 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (earliest_audio);
149                 Audio (tb.audio, tb.time);
150                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
151         }
152
153         /* Emit the earliest thing */
154
155         shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
156         shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
157         shared_ptr<DecodedSubtitle> ds = dynamic_pointer_cast<DecodedSubtitle> (earliest_decoded);
158
159         if (dv) {
160                 if (!_just_did_inaccurate_seek && earliest_time > _video_position) {
161                         /* XXX: if we're inside some content, repeat the last frame... otherwise emit black */
162                         emit_black ();
163                 } else {
164                         emit_video (earliest_piece, dv);
165                         earliest_piece->decoder->get ();
166                 }
167         } else if (da) {
168                 if (!_just_did_inaccurate_seek && earliest_time > _audio_position) {
169                         emit_silence (earliest_time - _audio_position);
170                 } else {
171                         emit_audio (earliest_piece, da);
172                         earliest_piece->decoder->get ();
173                 }
174         } else if (ds) {
175                 _in_subtitle.piece = earliest_piece;
176                 _in_subtitle.subtitle = ds;
177                 update_subtitle ();
178                 earliest_piece->decoder->get ();
179         }
180
181         _just_did_inaccurate_seek = false;
182
183         return false;
184 }
185
186 void
187 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
188 {
189         /* Keep a note of what came in so that we can repeat it if required */
190         _last_incoming_video.weak_piece = weak_piece;
191         _last_incoming_video.video = video;
192         
193         shared_ptr<Piece> piece = weak_piece.lock ();
194         if (!piece) {
195                 return;
196         }
197
198         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
199         assert (content);
200
201         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
202 #if 0
203         XXX
204         if (frc.skip && (frame % 2) == 1) {
205                 return;
206         }
207 #endif  
208
209         if (content->trimmed (video->dcp_time - content->position ())) {
210                 return;
211         }
212
213         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
214         libdcp::Size const image_size = fit_ratio_within (ratio, _video_container_size);
215
216         shared_ptr<PlayerImage> pi (
217                 new PlayerImage (
218                         video->image,
219                         content->crop(),
220                         image_size,
221                         _video_container_size,
222                         _film->scaler()
223                         )
224                 );
225         
226         if (
227                 _film->with_subtitles () &&
228                 _out_subtitle.subtitle->image &&
229                 video->dcp_time >= _out_subtitle.subtitle->dcp_time && video->dcp_time <= _out_subtitle.subtitle->dcp_time_to
230                 ) {
231
232                 Position<int> const container_offset (
233                         (_video_container_size.width - image_size.width) / 2,
234                         (_video_container_size.height - image_size.width) / 2
235                         );
236
237                 pi->set_subtitle (_out_subtitle.subtitle->image, _out_subtitle.position + container_offset);
238         }
239                                             
240 #ifdef DCPOMATIC_DEBUG
241         _last_video = piece->content;
242 #endif
243
244         Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
245
246         _last_emit_was_black = false;
247         _video_position = rint (video->dcp_time + TIME_HZ / _film->video_frame_rate());
248 }
249
250 void
251 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
252 {
253         shared_ptr<Piece> piece = weak_piece.lock ();
254         if (!piece) {
255                 return;
256         }
257
258         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
259         assert (content);
260
261         /* Gain */
262         if (content->audio_gain() != 0) {
263                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
264                 gain->apply_gain (content->audio_gain ());
265                 audio->data = gain;
266         }
267
268         /* Resample */
269         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
270                 audio->data = resampler(content, true)->run (audio->data);
271         }
272         
273         if (content->trimmed (audio->dcp_time - content->position ())) {
274                 return;
275         }
276
277         /* Remap channels */
278         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
279         dcp_mapped->make_silent ();
280         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
281         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
282                 if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) {
283                         dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second);
284                 }
285         }
286
287         audio->data = dcp_mapped;
288
289         /* Delay */
290         audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
291         if (audio->dcp_time < 0) {
292                 int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
293                 if (frames >= audio->data->frames ()) {
294                         return;
295                 }
296
297                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
298                 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
299
300                 audio->data = trimmed;
301                 audio->dcp_time = 0;
302         }
303
304         _audio_merger.push (audio->data, audio->dcp_time);
305 }
306
307 void
308 Player::flush ()
309 {
310         TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
311         if (tb.audio) {
312                 Audio (tb.audio, tb.time);
313                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
314         }
315
316         while (_video_position < _audio_position) {
317                 emit_black ();
318         }
319
320         while (_audio_position < _video_position) {
321                 emit_silence (_video_position - _audio_position);
322         }
323         
324 }
325
326 /** Seek so that the next pass() will yield (approximately) the requested frame.
327  *  Pass accurate = true to try harder to get close to the request.
328  *  @return true on error
329  */
330 void
331 Player::seek (DCPTime t, bool accurate)
332 {
333         if (!_have_valid_pieces) {
334                 setup_pieces ();
335         }
336
337         if (_pieces.empty ()) {
338                 return;
339         }
340
341         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
342                 /* s is the offset of t from the start position of this content */
343                 DCPTime s = t - (*i)->content->position ();
344                 s = max (static_cast<DCPTime> (0), s);
345                 s = min ((*i)->content->length_after_trim(), s);
346
347                 /* Convert this to the content time */
348                 ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start ();
349
350                 /* And seek the decoder */
351                 (*i)->decoder->seek (ct, accurate);
352         }
353
354         _video_position = _audio_position = t;
355         _audio_merger.clear (t);
356
357         if (!accurate) {
358                 /* We just did an inaccurate seek, so it's likely that the next thing seen
359                    out of pass() will be a fair distance from _{video,audio}_position.  Setting
360                    this flag stops pass() from trying to fix that: we assume that if it
361                    was an inaccurate seek then the caller does not care too much about
362                    inserting black/silence to keep the time tidy.
363                 */
364                 _just_did_inaccurate_seek = true;
365         }
366 }
367
368 void
369 Player::setup_pieces ()
370 {
371         list<shared_ptr<Piece> > old_pieces = _pieces;
372         _pieces.clear ();
373
374         ContentList content = _playlist->content ();
375
376         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
377
378                 shared_ptr<Decoder> decoder;
379                 optional<FrameRateChange> frc;
380
381                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
382                 if (fc) {
383                         decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
384                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
385                 }
386                 
387                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
388                 if (ic) {
389                         /* See if we can re-use an old ImageDecoder */
390                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
391                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
392                                 if (imd && imd->content() == ic) {
393                                         decoder = imd;
394                                 }
395                         }
396
397                         if (!decoder) {
398                                 decoder.reset (new ImageDecoder (_film, ic));
399                         }
400
401                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
402                 }
403
404                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
405                 if (sc) {
406                         decoder.reset (new SndfileDecoder (_film, sc));
407
408                         /* Working out the frc for this content is a bit tricky: what if it overlaps
409                            two pieces of video content with different frame rates?  For now, use
410                            the one with the best overlap.
411                         */
412
413                         DCPTime best_overlap_t = 0;
414                         shared_ptr<VideoContent> best_overlap;
415                         for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
416                                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
417                                 if (!vc) {
418                                         continue;
419                                 }
420
421                                 DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end());
422                                 if (overlap > best_overlap_t) {
423                                         best_overlap = vc;
424                                         best_overlap_t = overlap;
425                                 }
426                         }
427
428                         if (best_overlap) {
429                                 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
430                         } else {
431                                 /* No video overlap; e.g. if the DCP is just audio */
432                                 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
433                         }
434                 }
435
436                 decoder->seek ((*i)->trim_start (), true);
437                 
438                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
439         }
440
441         _have_valid_pieces = true;
442 }
443
444 void
445 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
446 {
447         shared_ptr<Content> c = w.lock ();
448         if (!c) {
449                 return;
450         }
451
452         if (
453                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
454                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
455                 ) {
456                 
457                 _have_valid_pieces = false;
458                 Changed (frequent);
459
460         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
461
462                 update_subtitle ();
463                 Changed (frequent);
464
465         } else if (
466                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
467                 property == VideoContentProperty::VIDEO_RATIO
468                 ) {
469                 
470                 Changed (frequent);
471
472         } else if (property == ContentProperty::PATH) {
473
474                 Changed (frequent);
475         }
476 }
477
478 void
479 Player::playlist_changed ()
480 {
481         _have_valid_pieces = false;
482         Changed (false);
483 }
484
485 void
486 Player::set_video_container_size (libdcp::Size s)
487 {
488         _video_container_size = s;
489
490         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
491         im->make_black ();
492         
493         _black_frame.reset (
494                 new PlayerImage (
495                         im,
496                         Crop(),
497                         _video_container_size,
498                         _video_container_size,
499                         Scaler::from_id ("bicubic")
500                         )
501                 );
502 }
503
504 shared_ptr<Resampler>
505 Player::resampler (shared_ptr<AudioContent> c, bool create)
506 {
507         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
508         if (i != _resamplers.end ()) {
509                 return i->second;
510         }
511
512         if (!create) {
513                 return shared_ptr<Resampler> ();
514         }
515
516         _film->log()->log (
517                 String::compose (
518                         "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
519                         )
520                 );
521         
522         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
523         _resamplers[c] = r;
524         return r;
525 }
526
527 void
528 Player::emit_black ()
529 {
530 #ifdef DCPOMATIC_DEBUG
531         _last_video.reset ();
532 #endif
533
534         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
535         _video_position += _film->video_frames_to_time (1);
536         _last_emit_was_black = true;
537 }
538
539 void
540 Player::emit_silence (DCPTime most)
541 {
542         if (most == 0) {
543                 return;
544         }
545         
546         DCPTime t = min (most, TIME_HZ / 2);
547         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ));
548         silence->make_silent ();
549         Audio (silence, _audio_position);
550         _audio_position += t;
551 }
552
553 void
554 Player::film_changed (Film::Property p)
555 {
556         /* Here we should notice Film properties that affect our output, and
557            alert listeners that our output now would be different to how it was
558            last time we were run.
559         */
560
561         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
562                 Changed (false);
563         }
564 }
565
566 void
567 Player::update_subtitle ()
568 {
569         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
570         if (!piece) {
571                 return;
572         }
573
574         if (!_in_subtitle.subtitle->image) {
575                 _out_subtitle.subtitle->image.reset ();
576                 return;
577         }
578
579         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
580         assert (sc);
581
582         dcpomatic::Rect<double> in_rect = _in_subtitle.subtitle->rect;
583         libdcp::Size scaled_size;
584
585         in_rect.y += sc->subtitle_offset ();
586
587         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
588         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
589         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
590
591         /* Then we need a corrective translation, consisting of two parts:
592          *
593          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
594          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
595          *
596          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
597          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
598          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
599          *
600          * Combining these two translations gives these expressions.
601          */
602         
603         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
604         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
605         
606         _out_subtitle.subtitle->image = _in_subtitle.subtitle->image->scale (
607                 scaled_size,
608                 Scaler::from_id ("bicubic"),
609                 _in_subtitle.subtitle->image->pixel_format (),
610                 true
611                 );
612         
613         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
614         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
615 }
616
617 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
618  *  @return false if this could not be done.
619  */
620 bool
621 Player::repeat_last_video ()
622 {
623         if (!_last_incoming_video.video || !_have_valid_pieces) {
624                 return false;
625         }
626
627         emit_video (
628                 _last_incoming_video.weak_piece,
629                 _last_incoming_video.video
630                 );
631
632         return true;
633 }
634
635 PlayerImage::PlayerImage (
636         shared_ptr<const Image> in,
637         Crop crop,
638         libdcp::Size inter_size,
639         libdcp::Size out_size,
640         Scaler const * scaler
641         )
642         : _in (in)
643         , _crop (crop)
644         , _inter_size (inter_size)
645         , _out_size (out_size)
646         , _scaler (scaler)
647 {
648
649 }
650
651 void
652 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
653 {
654         _subtitle_image = image;
655         _subtitle_position = pos;
656 }
657
658 shared_ptr<Image>
659 PlayerImage::image ()
660 {
661         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, PIX_FMT_RGB24, false);
662
663         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
664
665         if (_subtitle_image) {
666                 out->alpha_blend (_subtitle_image, _subtitle_position);
667         }
668
669         return out;
670 }