Fix audio weirdness when start-trimming.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "still_image_decoder.h"
26 #include "still_image_content.h"
27 #include "moving_image_decoder.h"
28 #include "moving_image_content.h"
29 #include "sndfile_decoder.h"
30 #include "sndfile_content.h"
31 #include "subtitle_content.h"
32 #include "playlist.h"
33 #include "job.h"
34 #include "image.h"
35 #include "ratio.h"
36 #include "resampler.h"
37 #include "log.h"
38 #include "scaler.h"
39
40 using std::list;
41 using std::cout;
42 using std::min;
43 using std::max;
44 using std::vector;
45 using std::pair;
46 using std::map;
47 using boost::shared_ptr;
48 using boost::weak_ptr;
49 using boost::dynamic_pointer_cast;
50
51 class Piece
52 {
53 public:
54         Piece (shared_ptr<Content> c)
55                 : content (c)
56                 , video_position (c->position ())
57                 , audio_position (c->position ())
58                 , repeat_to_do (0)
59                 , repeat_done (0)
60         {}
61         
62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
63                 : content (c)
64                 , decoder (d)
65                 , video_position (c->position ())
66                 , audio_position (c->position ())
67         {}
68
69         /** Set this piece to repeat a video frame a given number of times */
70         void set_repeat (IncomingVideo video, int num)
71         {
72                 repeat_video = video;
73                 repeat_to_do = num;
74                 repeat_done = 0;
75         }
76
77         void reset_repeat ()
78         {
79                 repeat_video.image.reset ();
80                 repeat_to_do = 0;
81                 repeat_done = 0;
82         }
83
84         bool repeating () const
85         {
86                 return repeat_done != repeat_to_do;
87         }
88
89         void repeat (Player* player)
90         {
91                 player->process_video (
92                         repeat_video.weak_piece,
93                         repeat_video.image,
94                         repeat_video.eyes,
95                         repeat_done > 0,
96                         repeat_video.frame,
97                         (repeat_done + 1) * (TIME_HZ / player->_film->video_frame_rate ())
98                         );
99
100                 ++repeat_done;
101         }
102         
103         shared_ptr<Content> content;
104         shared_ptr<Decoder> decoder;
105         Time video_position;
106         Time audio_position;
107
108         IncomingVideo repeat_video;
109         int repeat_to_do;
110         int repeat_done;
111 };
112
113 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
114         : _film (f)
115         , _playlist (p)
116         , _video (true)
117         , _audio (true)
118         , _have_valid_pieces (false)
119         , _video_position (0)
120         , _audio_position (0)
121         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
122         , _last_emit_was_black (false)
123 {
124         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
125         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
126         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
127         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
128 }
129
130 void
131 Player::disable_video ()
132 {
133         _video = false;
134 }
135
136 void
137 Player::disable_audio ()
138 {
139         _audio = false;
140 }
141
142 bool
143 Player::pass ()
144 {
145         if (!_have_valid_pieces) {
146                 setup_pieces ();
147                 _have_valid_pieces = true;
148         }
149
150         Time earliest_t = TIME_MAX;
151         shared_ptr<Piece> earliest;
152         enum {
153                 VIDEO,
154                 AUDIO
155         } type = VIDEO;
156
157         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
158                 if ((*i)->decoder->done ()) {
159                         continue;
160                 }
161
162                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
163                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
164
165                 if (_video && vd) {
166                         if ((*i)->video_position < earliest_t) {
167                                 earliest_t = (*i)->video_position;
168                                 earliest = *i;
169                                 type = VIDEO;
170                         }
171                 }
172
173                 if (_audio && ad && ad->has_audio ()) {
174                         if ((*i)->audio_position < earliest_t) {
175                                 earliest_t = (*i)->audio_position;
176                                 earliest = *i;
177                                 type = AUDIO;
178                         }
179                 }
180         }
181
182         if (!earliest) {
183                 flush ();
184                 return true;
185         }
186
187         switch (type) {
188         case VIDEO:
189                 if (earliest_t > _video_position) {
190                         emit_black ();
191                 } else {
192                         if (earliest->repeating ()) {
193                                 earliest->repeat (this);
194                         } else {
195                                 earliest->decoder->pass ();
196                         }
197                 }
198                 break;
199
200         case AUDIO:
201                 if (earliest_t > _audio_position) {
202                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
203                 } else {
204                         earliest->decoder->pass ();
205
206                         if (earliest->decoder->done()) {
207                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
208                                 assert (ac);
209                                 shared_ptr<Resampler> re = resampler (ac, false);
210                                 if (re) {
211                                         shared_ptr<const AudioBuffers> b = re->flush ();
212                                         if (b->frames ()) {
213                                                 process_audio (earliest, b, ac->audio_length ());
214                                         }
215                                 }
216                         }
217                 }
218                 break;
219         }
220
221         if (_audio) {
222                 Time audio_done_up_to = TIME_MAX;
223                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
224                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
225                                 audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
226                         }
227                 }
228
229                 TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
230                 Audio (tb.audio, tb.time);
231                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
232         }
233                 
234         return false;
235 }
236
237 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
238 void
239 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame, Time extra)
240 {
241         /* Keep a note of what came in so that we can repeat it if required */
242         _last_incoming_video.weak_piece = weak_piece;
243         _last_incoming_video.image = image;
244         _last_incoming_video.eyes = eyes;
245         _last_incoming_video.same = same;
246         _last_incoming_video.frame = frame;
247         _last_incoming_video.extra = extra;
248         
249         shared_ptr<Piece> piece = weak_piece.lock ();
250         if (!piece) {
251                 return;
252         }
253
254         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
255         assert (content);
256
257         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
258         if (frc.skip && (frame % 2) == 1) {
259                 return;
260         }
261
262         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
263         if (content->trimmed (relative_time)) {
264                 return;
265         }
266
267         /* Convert to RGB first, as FFmpeg doesn't seem to like handling YUV images with odd widths */
268         shared_ptr<Image> work_image = image->scale (image->size (), _film->scaler(), PIX_FMT_RGB24, true);
269
270         work_image = work_image->crop (content->crop(), true);
271
272         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
273         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
274         
275         work_image = work_image->scale (image_size, _film->scaler(), PIX_FMT_RGB24, true);
276
277         Time time = content->position() + relative_time + extra - content->trim_start ();
278             
279         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
280                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
281         }
282
283         if (image_size != _video_container_size) {
284                 assert (image_size.width <= _video_container_size.width);
285                 assert (image_size.height <= _video_container_size.height);
286                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
287                 im->make_black ();
288                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
289                 work_image = im;
290         }
291
292 #ifdef DCPOMATIC_DEBUG
293         _last_video = piece->content;
294 #endif
295
296         Video (work_image, eyes, content->colour_conversion(), same, time);
297
298         time += TIME_HZ / _film->video_frame_rate();
299         _last_emit_was_black = false;
300         _video_position = piece->video_position = time;
301
302         if (frc.repeat > 1 && !piece->repeating ()) {
303                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
304         }
305 }
306
307 void
308 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
309 {
310         shared_ptr<Piece> piece = weak_piece.lock ();
311         if (!piece) {
312                 return;
313         }
314
315         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
316         assert (content);
317
318         /* Gain */
319         if (content->audio_gain() != 0) {
320                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
321                 gain->apply_gain (content->audio_gain ());
322                 audio = gain;
323         }
324
325         /* Resample */
326         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
327                 shared_ptr<Resampler> r = resampler (content, true);
328                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
329                 audio = ro.first;
330                 frame = ro.second;
331         }
332         
333         Time const relative_time = _film->audio_frames_to_time (frame);
334
335         if (content->trimmed (relative_time)) {
336                 return;
337         }
338
339         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
340         
341         /* Remap channels */
342         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
343         dcp_mapped->make_silent ();
344         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
345         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
346                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
347                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
348                 }
349         }
350
351         audio = dcp_mapped;
352
353         /* We must cut off anything that comes before the start of all time */
354         if (time < 0) {
355                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
356                 if (frames >= audio->frames ()) {
357                         return;
358                 }
359
360                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
361                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
362
363                 audio = trimmed;
364                 time = 0;
365         }
366
367         _audio_merger.push (audio, time);
368         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
369 }
370
371 void
372 Player::flush ()
373 {
374         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
375         if (tb.audio) {
376                 Audio (tb.audio, tb.time);
377                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
378         }
379
380         while (_video_position < _audio_position) {
381                 emit_black ();
382         }
383
384         while (_audio_position < _video_position) {
385                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
386         }
387         
388 }
389
390 /** Seek so that the next pass() will yield (approximately) the requested frame.
391  *  Pass accurate = true to try harder to get close to the request.
392  *  @return true on error
393  */
394 void
395 Player::seek (Time t, bool accurate)
396 {
397         if (!_have_valid_pieces) {
398                 setup_pieces ();
399                 _have_valid_pieces = true;
400         }
401
402         if (_pieces.empty ()) {
403                 return;
404         }
405
406         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
407                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
408                 if (!vc) {
409                         continue;
410                 }
411                 
412                 Time s = t - vc->position ();
413                 s = max (static_cast<Time> (0), s);
414                 s = min (vc->length_after_trim(), s);
415
416                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
417
418                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
419                 /* Here we are converting from time (in the DCP) to a frame number in the content.
420                    Hence we need to use the DCP's frame rate and the double/skip correction, not
421                    the source's rate.
422                 */
423                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
424                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
425
426                 (*i)->reset_repeat ();
427         }
428
429         _video_position = _audio_position = t;
430
431         /* XXX: don't seek audio because we don't need to... */
432 }
433
434 void
435 Player::setup_pieces ()
436 {
437         list<shared_ptr<Piece> > old_pieces = _pieces;
438
439         _pieces.clear ();
440
441         ContentList content = _playlist->content ();
442         sort (content.begin(), content.end(), ContentSorter ());
443
444         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
445
446                 shared_ptr<Piece> piece (new Piece (*i));
447
448                 /* XXX: into content? */
449
450                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
451                 if (fc) {
452                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
453                         
454                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4, 0));
455                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
456                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
457
458                         piece->decoder = fd;
459                 }
460                 
461                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
462                 if (ic) {
463                         shared_ptr<StillImageDecoder> id;
464                         
465                         /* See if we can re-use an old StillImageDecoder */
466                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
467                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
468                                 if (imd && imd->content() == ic) {
469                                         id = imd;
470                                 }
471                         }
472
473                         if (!id) {
474                                 id.reset (new StillImageDecoder (_film, ic));
475                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4, 0));
476                         }
477
478                         piece->decoder = id;
479                 }
480
481                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
482                 if (mc) {
483                         shared_ptr<MovingImageDecoder> md;
484
485                         if (!md) {
486                                 md.reset (new MovingImageDecoder (_film, mc));
487                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4, 0));
488                         }
489
490                         piece->decoder = md;
491                 }
492
493                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
494                 if (sc) {
495                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
496                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
497
498                         piece->decoder = sd;
499                 }
500
501                 _pieces.push_back (piece);
502         }
503 }
504
505 void
506 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
507 {
508         shared_ptr<Content> c = w.lock ();
509         if (!c) {
510                 return;
511         }
512
513         if (
514                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
515                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
516                 ) {
517                 
518                 _have_valid_pieces = false;
519                 Changed (frequent);
520
521         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
522
523                 update_subtitle ();
524                 Changed (frequent);
525
526         } else if (
527                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
528                 property == VideoContentProperty::VIDEO_RATIO
529                 ) {
530                 
531                 Changed (frequent);
532
533         } else if (property == ContentProperty::PATH) {
534
535                 Changed (frequent);
536         }
537 }
538
539 void
540 Player::playlist_changed ()
541 {
542         _have_valid_pieces = false;
543         Changed (false);
544 }
545
546 void
547 Player::set_video_container_size (libdcp::Size s)
548 {
549         _video_container_size = s;
550         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
551         _black_frame->make_black ();
552 }
553
554 shared_ptr<Resampler>
555 Player::resampler (shared_ptr<AudioContent> c, bool create)
556 {
557         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
558         if (i != _resamplers.end ()) {
559                 return i->second;
560         }
561
562         if (!create) {
563                 return shared_ptr<Resampler> ();
564         }
565         
566         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
567         _resamplers[c] = r;
568         return r;
569 }
570
571 void
572 Player::emit_black ()
573 {
574 #ifdef DCPOMATIC_DEBUG
575         _last_video.reset ();
576 #endif
577
578         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
579         _video_position += _film->video_frames_to_time (1);
580         _last_emit_was_black = true;
581 }
582
583 void
584 Player::emit_silence (OutputAudioFrame most)
585 {
586         if (most == 0) {
587                 return;
588         }
589         
590         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
591         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
592         silence->make_silent ();
593         Audio (silence, _audio_position);
594         _audio_position += _film->audio_frames_to_time (N);
595 }
596
597 void
598 Player::film_changed (Film::Property p)
599 {
600         /* Here we should notice Film properties that affect our output, and
601            alert listeners that our output now would be different to how it was
602            last time we were run.
603         */
604
605         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
606                 Changed (false);
607         }
608 }
609
610 void
611 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
612 {
613         _in_subtitle.piece = weak_piece;
614         _in_subtitle.image = image;
615         _in_subtitle.rect = rect;
616         _in_subtitle.from = from;
617         _in_subtitle.to = to;
618
619         update_subtitle ();
620 }
621
622 void
623 Player::update_subtitle ()
624 {
625         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
626         if (!piece) {
627                 return;
628         }
629
630         if (!_in_subtitle.image) {
631                 _out_subtitle.image.reset ();
632                 return;
633         }
634
635         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
636         assert (sc);
637
638         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
639         libdcp::Size scaled_size;
640
641         in_rect.y += sc->subtitle_offset ();
642
643         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
644         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
645         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
646
647         /* Then we need a corrective translation, consisting of two parts:
648          *
649          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
650          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
651          *
652          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
653          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
654          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
655          *
656          * Combining these two translations gives these expressions.
657          */
658         
659         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
660         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
661         
662         _out_subtitle.image = _in_subtitle.image->scale (
663                 scaled_size,
664                 Scaler::from_id ("bicubic"),
665                 _in_subtitle.image->pixel_format (),
666                 true
667                 );
668         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
669         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
670 }
671
672 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
673  *  @return false if this could not be done.
674  */
675 bool
676 Player::repeat_last_video ()
677 {
678         if (!_last_incoming_video.image) {
679                 return false;
680         }
681
682         process_video (
683                 _last_incoming_video.weak_piece,
684                 _last_incoming_video.image,
685                 _last_incoming_video.eyes,
686                 _last_incoming_video.same,
687                 _last_incoming_video.frame,
688                 _last_incoming_video.extra
689                 );
690
691         return true;
692 }