Various fixes to film viewer.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "playlist.h"
31 #include "job.h"
32 #include "image.h"
33 #include "ratio.h"
34 #include "log.h"
35 #include "scaler.h"
36
37 using std::list;
38 using std::cout;
39 using std::min;
40 using std::max;
41 using std::vector;
42 using std::pair;
43 using std::map;
44 using boost::shared_ptr;
45 using boost::weak_ptr;
46 using boost::dynamic_pointer_cast;
47 using boost::optional;
48
49 class Piece
50 {
51 public:
52         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
53                 : content (c)
54                 , decoder (d)
55                 , frc (f)
56         {}
57
58         shared_ptr<Content> content;
59         shared_ptr<Decoder> decoder;
60         FrameRateChange frc;
61 };
62
63 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
64         : _film (f)
65         , _playlist (p)
66         , _video (true)
67         , _audio (true)
68         , _have_valid_pieces (false)
69         , _video_position (0)
70         , _audio_position (0)
71         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
72         , _last_emit_was_black (false)
73         , _just_did_inaccurate_seek (false)
74         , _approximate_size (false)
75 {
76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
79         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
80 }
81
82 void
83 Player::disable_video ()
84 {
85         _video = false;
86 }
87
88 void
89 Player::disable_audio ()
90 {
91         _audio = false;
92 }
93
94 bool
95 Player::pass ()
96 {
97         if (!_have_valid_pieces) {
98                 setup_pieces ();
99         }
100
101         /* Interrogate all our pieces to find the one with the earliest decoded data */
102
103         shared_ptr<Piece> earliest_piece;
104         shared_ptr<Decoded> earliest_decoded;
105         DCPTime earliest_time = TIME_MAX;
106         DCPTime earliest_audio = TIME_MAX;
107
108         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
109
110                 shared_ptr<Decoded> dec = (*i)->decoder->peek ();
111
112                 if (dec) {
113                         dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position());
114                 }
115
116                 if (dec && dec->dcp_time < earliest_time) {
117                         earliest_piece = *i;
118                         earliest_decoded = dec;
119                         earliest_time = dec->dcp_time;
120                 }
121
122                 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
123                         earliest_audio = dec->dcp_time;
124                 }
125         }
126                 
127         if (!earliest_piece) {
128                 flush ();
129                 return true;
130         }
131
132         if (earliest_audio != TIME_MAX) {
133                 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (earliest_audio);
134                 Audio (tb.audio, tb.time);
135                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
136         }
137
138         /* Emit the earliest thing */
139
140         shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
141         shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
142         shared_ptr<DecodedSubtitle> ds = dynamic_pointer_cast<DecodedSubtitle> (earliest_decoded);
143
144         if (dv && _video) {
145                 DCPTime const half_frame = TIME_HZ / (2 * _film->video_frame_rate ());
146
147                 if (_just_did_inaccurate_seek) {
148                         /* Just emit; no subtlety */
149                         emit_video (earliest_piece, dv);
150                         earliest_piece->decoder->get ();
151                 } else if (earliest_time > (_video_position + half_frame)) {
152
153                         /* See if we're inside some video content */
154                         list<shared_ptr<Piece> >::iterator i = _pieces.begin();
155                         while (i != _pieces.end() && ((*i)->content->position() >= _video_position || _video_position >= (*i)->content->end())) {
156                                 ++i;
157                         }
158
159                         if (i == _pieces.end() || !_last_incoming_video.video || !_have_valid_pieces) {
160                                 /* We're outside all video content */
161                                 emit_black ();
162                         } else {
163                                 _last_incoming_video.video->dcp_time = _video_position;
164                                 emit_video (_last_incoming_video.weak_piece, _last_incoming_video.video);
165                         }
166
167                 } else {
168
169                         if (
170                                 dv->dcp_time >= _video_position &&
171                                 !earliest_piece->content->trimmed (dv->dcp_time - earliest_piece->content->position ())
172                                 ) {
173
174                                 emit_video (earliest_piece, dv);
175                         }
176                 
177                         earliest_piece->decoder->get ();
178                 }
179
180         } else if (da && _audio) {
181                 if (!_just_did_inaccurate_seek && earliest_time > _audio_position) {
182                         emit_silence (earliest_time - _audio_position);
183                 } else {
184                         emit_audio (earliest_piece, da);
185                         earliest_piece->decoder->get ();
186                 }
187         } else if (ds && _video) {
188                 _in_subtitle.piece = earliest_piece;
189                 _in_subtitle.subtitle = ds;
190                 update_subtitle ();
191                 earliest_piece->decoder->get ();
192         }
193
194         _just_did_inaccurate_seek = false;
195
196         return false;
197 }
198
199 void
200 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
201 {
202         /* Keep a note of what came in so that we can repeat it if required */
203         _last_incoming_video.weak_piece = weak_piece;
204         _last_incoming_video.video = video;
205         
206         shared_ptr<Piece> piece = weak_piece.lock ();
207         if (!piece) {
208                 return;
209         }
210
211         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
212         assert (content);
213
214         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
215
216         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
217         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
218         if (_approximate_size) {
219                 image_size.width &= ~3;
220                 image_size.height &= ~3;
221         }
222
223         shared_ptr<PlayerImage> pi (
224                 new PlayerImage (
225                         video->image,
226                         content->crop(),
227                         image_size,
228                         _video_container_size,
229                         _film->scaler()
230                         )
231                 );
232         
233         if (
234                 _film->with_subtitles () &&
235                 _out_subtitle.subtitle->image &&
236                 video->dcp_time >= _out_subtitle.subtitle->dcp_time && video->dcp_time <= _out_subtitle.subtitle->dcp_time_to
237                 ) {
238
239                 Position<int> const container_offset (
240                         (_video_container_size.width - image_size.width) / 2,
241                         (_video_container_size.height - image_size.width) / 2
242                         );
243
244                 pi->set_subtitle (_out_subtitle.subtitle->image, _out_subtitle.position + container_offset);
245         }
246                                             
247 #ifdef DCPOMATIC_DEBUG
248         _last_video = piece->content;
249 #endif
250
251         Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
252         
253         _last_emit_was_black = false;
254
255         /* This is a bit of a hack; don't update _video_position if EYES_RIGHT is on its way */
256         if (video->eyes != EYES_LEFT) {
257                 _video_position = rint (video->dcp_time + TIME_HZ / _film->video_frame_rate());
258         }
259 }
260
261 void
262 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
263 {
264         shared_ptr<Piece> piece = weak_piece.lock ();
265         if (!piece) {
266                 return;
267         }
268
269         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
270         assert (content);
271
272         /* Gain */
273         if (content->audio_gain() != 0) {
274                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
275                 gain->apply_gain (content->audio_gain ());
276                 audio->data = gain;
277         }
278
279         if (content->trimmed (audio->dcp_time - content->position ())) {
280                 return;
281         }
282
283         /* Remap channels */
284         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
285         dcp_mapped->make_silent ();
286         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
287         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
288                 if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) {
289                         dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second);
290                 }
291         }
292
293         audio->data = dcp_mapped;
294
295         /* Delay */
296         audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
297         if (audio->dcp_time < 0) {
298                 int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
299                 if (frames >= audio->data->frames ()) {
300                         return;
301                 }
302
303                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
304                 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
305
306                 audio->data = trimmed;
307                 audio->dcp_time = 0;
308         }
309
310         _audio_merger.push (audio->data, audio->dcp_time);
311 }
312
313 void
314 Player::flush ()
315 {
316         TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
317         if (tb.audio) {
318                 Audio (tb.audio, tb.time);
319                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
320         }
321
322         while (_video_position < _audio_position) {
323                 emit_black ();
324         }
325
326         while (_audio_position < _video_position) {
327                 emit_silence (_video_position - _audio_position);
328         }
329         
330 }
331
332 /** Seek so that the next pass() will yield (approximately) the requested frame.
333  *  Pass accurate = true to try harder to get close to the request.
334  *  @return true on error
335  */
336 void
337 Player::seek (DCPTime t, bool accurate)
338 {
339         if (!_have_valid_pieces) {
340                 setup_pieces ();
341         }
342
343         if (_pieces.empty ()) {
344                 return;
345         }
346
347         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
348                 /* s is the offset of t from the start position of this content */
349                 DCPTime s = t - (*i)->content->position ();
350                 s = max (static_cast<DCPTime> (0), s);
351                 s = min ((*i)->content->length_after_trim(), s);
352
353                 /* Convert this to the content time */
354                 ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start ();
355
356                 /* And seek the decoder */
357                 (*i)->decoder->seek (ct, accurate);
358         }
359
360         _video_position = time_round_up (t, TIME_HZ / _film->video_frame_rate());
361         _audio_position = time_round_up (t, TIME_HZ / _film->audio_frame_rate());
362
363         _audio_merger.clear (_audio_position);
364
365         if (!accurate) {
366                 /* We just did an inaccurate seek, so it's likely that the next thing seen
367                    out of pass() will be a fair distance from _{video,audio}_position.  Setting
368                    this flag stops pass() from trying to fix that: we assume that if it
369                    was an inaccurate seek then the caller does not care too much about
370                    inserting black/silence to keep the time tidy.
371                 */
372                 _just_did_inaccurate_seek = true;
373         }
374 }
375
376 void
377 Player::setup_pieces ()
378 {
379         list<shared_ptr<Piece> > old_pieces = _pieces;
380         _pieces.clear ();
381
382         ContentList content = _playlist->content ();
383
384         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
385
386                 shared_ptr<Decoder> decoder;
387                 optional<FrameRateChange> frc;
388
389                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
390                 if (fc) {
391                         decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
392                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
393                 }
394                 
395                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
396                 if (ic) {
397                         /* See if we can re-use an old ImageDecoder */
398                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
399                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
400                                 if (imd && imd->content() == ic) {
401                                         decoder = imd;
402                                 }
403                         }
404
405                         if (!decoder) {
406                                 decoder.reset (new ImageDecoder (_film, ic));
407                         }
408
409                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
410                 }
411
412                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
413                 if (sc) {
414                         decoder.reset (new SndfileDecoder (_film, sc));
415
416                         /* Working out the frc for this content is a bit tricky: what if it overlaps
417                            two pieces of video content with different frame rates?  For now, use
418                            the one with the best overlap.
419                         */
420
421                         DCPTime best_overlap_t = 0;
422                         shared_ptr<VideoContent> best_overlap;
423                         for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
424                                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
425                                 if (!vc) {
426                                         continue;
427                                 }
428
429                                 DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end());
430                                 if (overlap > best_overlap_t) {
431                                         best_overlap = vc;
432                                         best_overlap_t = overlap;
433                                 }
434                         }
435
436                         if (best_overlap) {
437                                 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
438                         } else {
439                                 /* No video overlap; e.g. if the DCP is just audio */
440                                 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
441                         }
442                 }
443
444                 decoder->seek ((*i)->trim_start (), true);
445                 
446                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
447         }
448
449         _have_valid_pieces = true;
450
451         /* The Piece for the _last_incoming_video will no longer be valid */
452         _last_incoming_video.video.reset ();
453
454         _video_position = _audio_position = 0;
455 }
456
457 void
458 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
459 {
460         shared_ptr<Content> c = w.lock ();
461         if (!c) {
462                 return;
463         }
464
465         if (
466                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
467                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
468                 property == VideoContentProperty::VIDEO_FRAME_TYPE 
469                 ) {
470                 
471                 _have_valid_pieces = false;
472                 Changed (frequent);
473
474         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
475
476                 update_subtitle ();
477                 Changed (frequent);
478
479         } else if (property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO) {
480                 
481                 Changed (frequent);
482
483         } else if (property == ContentProperty::PATH) {
484
485                 Changed (frequent);
486         }
487 }
488
489 void
490 Player::playlist_changed ()
491 {
492         _have_valid_pieces = false;
493         Changed (false);
494 }
495
496 void
497 Player::set_video_container_size (libdcp::Size s)
498 {
499         _video_container_size = s;
500
501         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
502         im->make_black ();
503         
504         _black_frame.reset (
505                 new PlayerImage (
506                         im,
507                         Crop(),
508                         _video_container_size,
509                         _video_container_size,
510                         Scaler::from_id ("bicubic")
511                         )
512                 );
513 }
514
515 void
516 Player::emit_black ()
517 {
518 #ifdef DCPOMATIC_DEBUG
519         _last_video.reset ();
520 #endif
521
522         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
523         _video_position += _film->video_frames_to_time (1);
524         _last_emit_was_black = true;
525 }
526
527 void
528 Player::emit_silence (DCPTime most)
529 {
530         if (most == 0) {
531                 return;
532         }
533         
534         DCPTime t = min (most, TIME_HZ / 2);
535         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ));
536         silence->make_silent ();
537         Audio (silence, _audio_position);
538         
539         _audio_position += t;
540 }
541
542 void
543 Player::film_changed (Film::Property p)
544 {
545         /* Here we should notice Film properties that affect our output, and
546            alert listeners that our output now would be different to how it was
547            last time we were run.
548         */
549
550         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
551                 Changed (false);
552         }
553 }
554
555 void
556 Player::update_subtitle ()
557 {
558         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
559         if (!piece) {
560                 return;
561         }
562
563         if (!_in_subtitle.subtitle->image) {
564                 _out_subtitle.subtitle->image.reset ();
565                 return;
566         }
567
568         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
569         assert (sc);
570
571         dcpomatic::Rect<double> in_rect = _in_subtitle.subtitle->rect;
572         libdcp::Size scaled_size;
573
574         in_rect.y += sc->subtitle_offset ();
575
576         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
577         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
578         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
579
580         /* Then we need a corrective translation, consisting of two parts:
581          *
582          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
583          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
584          *
585          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
586          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
587          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
588          *
589          * Combining these two translations gives these expressions.
590          */
591         
592         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
593         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
594         
595         _out_subtitle.subtitle->image = _in_subtitle.subtitle->image->scale (
596                 scaled_size,
597                 Scaler::from_id ("bicubic"),
598                 _in_subtitle.subtitle->image->pixel_format (),
599                 true
600                 );
601         
602         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
603         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
604 }
605
606 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
607  *  @return false if this could not be done.
608  */
609 bool
610 Player::repeat_last_video ()
611 {
612         if (!_last_incoming_video.video || !_have_valid_pieces) {
613                 return false;
614         }
615
616         emit_video (
617                 _last_incoming_video.weak_piece,
618                 _last_incoming_video.video
619                 );
620
621         return true;
622 }
623
624 void
625 Player::set_approximate_size ()
626 {
627         _approximate_size = true;
628 }
629                               
630
631 PlayerImage::PlayerImage (
632         shared_ptr<const Image> in,
633         Crop crop,
634         libdcp::Size inter_size,
635         libdcp::Size out_size,
636         Scaler const * scaler
637         )
638         : _in (in)
639         , _crop (crop)
640         , _inter_size (inter_size)
641         , _out_size (out_size)
642         , _scaler (scaler)
643 {
644
645 }
646
647 void
648 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
649 {
650         _subtitle_image = image;
651         _subtitle_position = pos;
652 }
653
654 shared_ptr<Image>
655 PlayerImage::image (AVPixelFormat format, bool aligned)
656 {
657         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, format, aligned);
658         
659         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
660
661         if (_subtitle_image) {
662                 out->alpha_blend (_subtitle_image, _subtitle_position);
663         }
664
665         return out;
666 }
667