Merge branch '1.0' into 1.0-seek
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "playlist.h"
31 #include "job.h"
32 #include "image.h"
33 #include "ratio.h"
34 #include "resampler.h"
35 #include "log.h"
36 #include "scaler.h"
37
38 using std::list;
39 using std::cout;
40 using std::min;
41 using std::max;
42 using std::vector;
43 using std::pair;
44 using std::map;
45 using boost::shared_ptr;
46 using boost::weak_ptr;
47 using boost::dynamic_pointer_cast;
48 using boost::optional;
49
50 class Piece
51 {
52 public:
53         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
54                 : content (c)
55                 , decoder (d)
56                 , frc (f)
57         {}
58
59         shared_ptr<Content> content;
60         shared_ptr<Decoder> decoder;
61         FrameRateChange frc;
62 };
63
64 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
65         : _film (f)
66         , _playlist (p)
67         , _video (true)
68         , _audio (true)
69         , _have_valid_pieces (false)
70         , _video_position (0)
71         , _audio_position (0)
72         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
73         , _last_emit_was_black (false)
74         , _just_did_inaccurate_seek (false)
75 {
76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
79         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
80 }
81
82 void
83 Player::disable_video ()
84 {
85         _video = false;
86 }
87
88 void
89 Player::disable_audio ()
90 {
91         _audio = false;
92 }
93
94 bool
95 Player::pass ()
96 {
97         if (!_have_valid_pieces) {
98                 setup_pieces ();
99         }
100
101         /* Interrogate all our pieces to find the one with the earliest decoded data */
102
103         shared_ptr<Piece> earliest_piece;
104         shared_ptr<Decoded> earliest_decoded;
105         DCPTime earliest_time = TIME_MAX;
106         DCPTime earliest_audio = TIME_MAX;
107
108         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
109
110                 shared_ptr<Decoded> dec = (*i)->decoder->peek ();
111
112                 if (dec) {
113                         dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position());
114                 }
115
116                 /* XXX: don't know what to do with this */
117 #if 0           
118                 if (ad->done()) {
119                         shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> ((*i)->content);
120                         assert (ac);
121                         shared_ptr<Resampler> re = resampler (ac, false);
122                         if (re) {
123                                 shared_ptr<const AudioBuffers> b = re->flush ();
124                                 if (b->frames ()) {
125                                         process_audio (earliest, b, ac->audio_length ());
126                                 }
127                         }
128                 }
129 #endif          
130
131                 if (dec && dec->dcp_time < earliest_time) {
132                         earliest_piece = *i;
133                         earliest_decoded = dec;
134                         earliest_time = dec->dcp_time;
135                 }
136
137                 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
138                         earliest_audio = dec->dcp_time;
139                 }
140         }
141                 
142         if (!earliest_piece) {
143                 flush ();
144                 return true;
145         }
146
147         if (earliest_audio != TIME_MAX) {
148                 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (earliest_audio);
149                 Audio (tb.audio, tb.time);
150                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
151         }
152
153         /* Emit the earliest thing */
154
155         shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
156         shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
157         shared_ptr<DecodedSubtitle> ds = dynamic_pointer_cast<DecodedSubtitle> (earliest_decoded);
158
159         if (dv) {
160                 if (!_just_did_inaccurate_seek && earliest_time > _video_position) {
161
162                         /* See if we're inside some video content */
163                         list<shared_ptr<Piece> >::iterator i = _pieces.begin();
164                         while (i != _pieces.end() && ((*i)->content->position() >= _video_position || _video_position >= (*i)->content->end())) {
165                                 ++i;
166                         }
167
168                         if (i == _pieces.end() || !_last_incoming_video.video || !_have_valid_pieces) {
169                                 /* We're outside all video content */
170                                 emit_black ();
171                         } else {
172                                 _last_incoming_video.video->dcp_time = _video_position;
173                                 emit_video (_last_incoming_video.weak_piece, _last_incoming_video.video);
174                         }
175                 } else {
176                         emit_video (earliest_piece, dv);
177                         earliest_piece->decoder->get ();
178                 }
179         } else if (da) {
180                 if (!_just_did_inaccurate_seek && earliest_time > _audio_position) {
181                         emit_silence (earliest_time - _audio_position);
182                 } else {
183                         emit_audio (earliest_piece, da);
184                         earliest_piece->decoder->get ();
185                 }
186         } else if (ds) {
187                 _in_subtitle.piece = earliest_piece;
188                 _in_subtitle.subtitle = ds;
189                 update_subtitle ();
190                 earliest_piece->decoder->get ();
191         }
192
193         _just_did_inaccurate_seek = false;
194
195         return false;
196 }
197
198 void
199 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
200 {
201         /* Keep a note of what came in so that we can repeat it if required */
202         _last_incoming_video.weak_piece = weak_piece;
203         _last_incoming_video.video = video;
204         
205         shared_ptr<Piece> piece = weak_piece.lock ();
206         if (!piece) {
207                 return;
208         }
209
210         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
211         assert (content);
212
213         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
214 #if 0
215         XXX
216         if (frc.skip && (frame % 2) == 1) {
217                 return;
218         }
219 #endif  
220
221         if (content->trimmed (video->dcp_time - content->position ())) {
222                 return;
223         }
224
225         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
226         libdcp::Size const image_size = fit_ratio_within (ratio, _video_container_size);
227
228         shared_ptr<PlayerImage> pi (
229                 new PlayerImage (
230                         video->image,
231                         content->crop(),
232                         image_size,
233                         _video_container_size,
234                         _film->scaler()
235                         )
236                 );
237         
238         if (
239                 _film->with_subtitles () &&
240                 _out_subtitle.subtitle->image &&
241                 video->dcp_time >= _out_subtitle.subtitle->dcp_time && video->dcp_time <= _out_subtitle.subtitle->dcp_time_to
242                 ) {
243
244                 Position<int> const container_offset (
245                         (_video_container_size.width - image_size.width) / 2,
246                         (_video_container_size.height - image_size.width) / 2
247                         );
248
249                 pi->set_subtitle (_out_subtitle.subtitle->image, _out_subtitle.position + container_offset);
250         }
251                                             
252 #ifdef DCPOMATIC_DEBUG
253         _last_video = piece->content;
254 #endif
255
256         Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
257
258         _last_emit_was_black = false;
259         _video_position = rint (video->dcp_time + TIME_HZ / _film->video_frame_rate());
260 }
261
262 void
263 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
264 {
265         shared_ptr<Piece> piece = weak_piece.lock ();
266         if (!piece) {
267                 return;
268         }
269
270         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
271         assert (content);
272
273         /* Gain */
274         if (content->audio_gain() != 0) {
275                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
276                 gain->apply_gain (content->audio_gain ());
277                 audio->data = gain;
278         }
279
280         /* Resample */
281         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
282                 audio->data = resampler(content, true)->run (audio->data);
283         }
284         
285         if (content->trimmed (audio->dcp_time - content->position ())) {
286                 return;
287         }
288
289         /* Remap channels */
290         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
291         dcp_mapped->make_silent ();
292         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
293         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
294                 if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) {
295                         dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second);
296                 }
297         }
298
299         audio->data = dcp_mapped;
300
301         /* Delay */
302         audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
303         if (audio->dcp_time < 0) {
304                 int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
305                 if (frames >= audio->data->frames ()) {
306                         return;
307                 }
308
309                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
310                 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
311
312                 audio->data = trimmed;
313                 audio->dcp_time = 0;
314         }
315
316         _audio_merger.push (audio->data, audio->dcp_time);
317 }
318
319 void
320 Player::flush ()
321 {
322         TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
323         if (tb.audio) {
324                 Audio (tb.audio, tb.time);
325                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
326         }
327
328         while (_video_position < _audio_position) {
329                 emit_black ();
330         }
331
332         while (_audio_position < _video_position) {
333                 emit_silence (_video_position - _audio_position);
334         }
335         
336 }
337
338 /** Seek so that the next pass() will yield (approximately) the requested frame.
339  *  Pass accurate = true to try harder to get close to the request.
340  *  @return true on error
341  */
342 void
343 Player::seek (DCPTime t, bool accurate)
344 {
345         if (!_have_valid_pieces) {
346                 setup_pieces ();
347         }
348
349         if (_pieces.empty ()) {
350                 return;
351         }
352
353         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
354                 /* s is the offset of t from the start position of this content */
355                 DCPTime s = t - (*i)->content->position ();
356                 s = max (static_cast<DCPTime> (0), s);
357                 s = min ((*i)->content->length_after_trim(), s);
358
359                 /* Convert this to the content time */
360                 ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start ();
361
362                 /* And seek the decoder */
363                 (*i)->decoder->seek (ct, accurate);
364         }
365
366         _video_position = time_round_up (t, TIME_HZ / _film->video_frame_rate());
367         _audio_position = time_round_up (t, TIME_HZ / _film->audio_frame_rate());
368
369         _audio_merger.clear (_audio_position);
370
371         if (!accurate) {
372                 /* We just did an inaccurate seek, so it's likely that the next thing seen
373                    out of pass() will be a fair distance from _{video,audio}_position.  Setting
374                    this flag stops pass() from trying to fix that: we assume that if it
375                    was an inaccurate seek then the caller does not care too much about
376                    inserting black/silence to keep the time tidy.
377                 */
378                 _just_did_inaccurate_seek = true;
379         }
380 }
381
382 void
383 Player::setup_pieces ()
384 {
385         list<shared_ptr<Piece> > old_pieces = _pieces;
386         _pieces.clear ();
387
388         ContentList content = _playlist->content ();
389
390         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
391
392                 shared_ptr<Decoder> decoder;
393                 optional<FrameRateChange> frc;
394
395                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
396                 if (fc) {
397                         decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
398                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
399                 }
400                 
401                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
402                 if (ic) {
403                         /* See if we can re-use an old ImageDecoder */
404                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
405                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
406                                 if (imd && imd->content() == ic) {
407                                         decoder = imd;
408                                 }
409                         }
410
411                         if (!decoder) {
412                                 decoder.reset (new ImageDecoder (_film, ic));
413                         }
414
415                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
416                 }
417
418                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
419                 if (sc) {
420                         decoder.reset (new SndfileDecoder (_film, sc));
421
422                         /* Working out the frc for this content is a bit tricky: what if it overlaps
423                            two pieces of video content with different frame rates?  For now, use
424                            the one with the best overlap.
425                         */
426
427                         DCPTime best_overlap_t = 0;
428                         shared_ptr<VideoContent> best_overlap;
429                         for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
430                                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
431                                 if (!vc) {
432                                         continue;
433                                 }
434
435                                 DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end());
436                                 if (overlap > best_overlap_t) {
437                                         best_overlap = vc;
438                                         best_overlap_t = overlap;
439                                 }
440                         }
441
442                         if (best_overlap) {
443                                 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
444                         } else {
445                                 /* No video overlap; e.g. if the DCP is just audio */
446                                 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
447                         }
448                 }
449
450                 decoder->seek ((*i)->trim_start (), true);
451                 
452                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
453         }
454
455         _have_valid_pieces = true;
456 }
457
458 void
459 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
460 {
461         shared_ptr<Content> c = w.lock ();
462         if (!c) {
463                 return;
464         }
465
466         if (
467                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
468                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
469                 property == VideoContentProperty::VIDEO_FRAME_TYPE 
470                 ) {
471                 
472                 _have_valid_pieces = false;
473                 Changed (frequent);
474
475         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
476
477                 update_subtitle ();
478                 Changed (frequent);
479
480         } else if (property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO) {
481                 
482                 Changed (frequent);
483
484         } else if (property == ContentProperty::PATH) {
485
486                 Changed (frequent);
487         }
488 }
489
490 void
491 Player::playlist_changed ()
492 {
493         _have_valid_pieces = false;
494         Changed (false);
495 }
496
497 void
498 Player::set_video_container_size (libdcp::Size s)
499 {
500         _video_container_size = s;
501
502         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
503         im->make_black ();
504         
505         _black_frame.reset (
506                 new PlayerImage (
507                         im,
508                         Crop(),
509                         _video_container_size,
510                         _video_container_size,
511                         Scaler::from_id ("bicubic")
512                         )
513                 );
514 }
515
516 shared_ptr<Resampler>
517 Player::resampler (shared_ptr<AudioContent> c, bool create)
518 {
519         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
520         if (i != _resamplers.end ()) {
521                 return i->second;
522         }
523
524         if (!create) {
525                 return shared_ptr<Resampler> ();
526         }
527
528         _film->log()->log (
529                 String::compose (
530                         "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
531                         )
532                 );
533         
534         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
535         _resamplers[c] = r;
536         return r;
537 }
538
539 void
540 Player::emit_black ()
541 {
542 #ifdef DCPOMATIC_DEBUG
543         _last_video.reset ();
544 #endif
545
546         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
547         _video_position += _film->video_frames_to_time (1);
548         _last_emit_was_black = true;
549 }
550
551 void
552 Player::emit_silence (DCPTime most)
553 {
554         if (most == 0) {
555                 return;
556         }
557         
558         DCPTime t = min (most, TIME_HZ / 2);
559         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ));
560         silence->make_silent ();
561         Audio (silence, _audio_position);
562         _audio_position += t;
563 }
564
565 void
566 Player::film_changed (Film::Property p)
567 {
568         /* Here we should notice Film properties that affect our output, and
569            alert listeners that our output now would be different to how it was
570            last time we were run.
571         */
572
573         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
574                 Changed (false);
575         }
576 }
577
578 void
579 Player::update_subtitle ()
580 {
581         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
582         if (!piece) {
583                 return;
584         }
585
586         if (!_in_subtitle.subtitle->image) {
587                 _out_subtitle.subtitle->image.reset ();
588                 return;
589         }
590
591         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
592         assert (sc);
593
594         dcpomatic::Rect<double> in_rect = _in_subtitle.subtitle->rect;
595         libdcp::Size scaled_size;
596
597         in_rect.y += sc->subtitle_offset ();
598
599         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
600         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
601         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
602
603         /* Then we need a corrective translation, consisting of two parts:
604          *
605          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
606          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
607          *
608          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
609          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
610          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
611          *
612          * Combining these two translations gives these expressions.
613          */
614         
615         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
616         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
617         
618         _out_subtitle.subtitle->image = _in_subtitle.subtitle->image->scale (
619                 scaled_size,
620                 Scaler::from_id ("bicubic"),
621                 _in_subtitle.subtitle->image->pixel_format (),
622                 true
623                 );
624         
625         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
626         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
627 }
628
629 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
630  *  @return false if this could not be done.
631  */
632 bool
633 Player::repeat_last_video ()
634 {
635         if (!_last_incoming_video.video || !_have_valid_pieces) {
636                 return false;
637         }
638
639         emit_video (
640                 _last_incoming_video.weak_piece,
641                 _last_incoming_video.video
642                 );
643
644         return true;
645 }
646
647 PlayerImage::PlayerImage (
648         shared_ptr<const Image> in,
649         Crop crop,
650         libdcp::Size inter_size,
651         libdcp::Size out_size,
652         Scaler const * scaler
653         )
654         : _in (in)
655         , _crop (crop)
656         , _inter_size (inter_size)
657         , _out_size (out_size)
658         , _scaler (scaler)
659 {
660
661 }
662
663 void
664 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
665 {
666         _subtitle_image = image;
667         _subtitle_position = pos;
668 }
669
670 shared_ptr<Image>
671 PlayerImage::image ()
672 {
673         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, PIX_FMT_RGB24, false);
674
675         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
676
677         if (_subtitle_image) {
678                 out->alpha_blend (_subtitle_image, _subtitle_position);
679         }
680
681         return out;
682 }