Do lots of the player processing with less copying.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "playlist.h"
31 #include "job.h"
32 #include "image.h"
33 #include "ratio.h"
34 #include "resampler.h"
35 #include "log.h"
36 #include "scaler.h"
37
38 using std::list;
39 using std::cout;
40 using std::min;
41 using std::max;
42 using std::vector;
43 using std::pair;
44 using std::map;
45 using boost::shared_ptr;
46 using boost::weak_ptr;
47 using boost::dynamic_pointer_cast;
48
49 class Piece
50 {
51 public:
52         Piece (shared_ptr<Content> c)
53                 : content (c)
54                 , video_position (c->position ())
55                 , audio_position (c->position ())
56                 , repeat_to_do (0)
57                 , repeat_done (0)
58         {}
59         
60         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
61                 : content (c)
62                 , decoder (d)
63                 , video_position (c->position ())
64                 , audio_position (c->position ())
65         {}
66
67         /** Set this piece to repeat a video frame a given number of times */
68         void set_repeat (IncomingVideo video, int num)
69         {
70                 repeat_video = video;
71                 repeat_to_do = num;
72                 repeat_done = 0;
73         }
74
75         void reset_repeat ()
76         {
77                 repeat_video.image.reset ();
78                 repeat_to_do = 0;
79                 repeat_done = 0;
80         }
81
82         bool repeating () const
83         {
84                 return repeat_done != repeat_to_do;
85         }
86
87         void repeat (Player* player)
88         {
89                 player->process_video (
90                         repeat_video.weak_piece,
91                         repeat_video.image,
92                         repeat_video.eyes,
93                         repeat_done > 0,
94                         repeat_video.frame,
95                         (repeat_done + 1) * (TIME_HZ / player->_film->video_frame_rate ())
96                         );
97
98                 ++repeat_done;
99         }
100         
101         shared_ptr<Content> content;
102         shared_ptr<Decoder> decoder;
103         /** Time of the last video we emitted relative to the start of the DCP */
104         Time video_position;
105         /** Time of the last audio we emitted relative to the start of the DCP */
106         Time audio_position;
107
108         IncomingVideo repeat_video;
109         int repeat_to_do;
110         int repeat_done;
111 };
112
113 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
114         : _film (f)
115         , _playlist (p)
116         , _video (true)
117         , _audio (true)
118         , _have_valid_pieces (false)
119         , _video_position (0)
120         , _audio_position (0)
121         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
122         , _last_emit_was_black (false)
123 {
124         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
125         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
126         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
127         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
128 }
129
130 void
131 Player::disable_video ()
132 {
133         _video = false;
134 }
135
136 void
137 Player::disable_audio ()
138 {
139         _audio = false;
140 }
141
142 bool
143 Player::pass ()
144 {
145         if (!_have_valid_pieces) {
146                 setup_pieces ();
147                 _have_valid_pieces = true;
148         }
149
150         Time earliest_t = TIME_MAX;
151         shared_ptr<Piece> earliest;
152         enum {
153                 VIDEO,
154                 AUDIO
155         } type = VIDEO;
156
157         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
158                 if ((*i)->decoder->done ()) {
159                         continue;
160                 }
161
162                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
163                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
164
165                 if (_video && vd) {
166                         if ((*i)->video_position < earliest_t) {
167                                 earliest_t = (*i)->video_position;
168                                 earliest = *i;
169                                 type = VIDEO;
170                         }
171                 }
172
173                 if (_audio && ad && ad->has_audio ()) {
174                         if ((*i)->audio_position < earliest_t) {
175                                 earliest_t = (*i)->audio_position;
176                                 earliest = *i;
177                                 type = AUDIO;
178                         }
179                 }
180         }
181
182         if (!earliest) {
183                 flush ();
184                 return true;
185         }
186
187         switch (type) {
188         case VIDEO:
189                 if (earliest_t > _video_position) {
190                         emit_black ();
191                 } else {
192                         if (earliest->repeating ()) {
193                                 earliest->repeat (this);
194                         } else {
195                                 earliest->decoder->pass ();
196                         }
197                 }
198                 break;
199
200         case AUDIO:
201                 if (earliest_t > _audio_position) {
202                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
203                 } else {
204                         earliest->decoder->pass ();
205
206                         if (earliest->decoder->done()) {
207                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
208                                 assert (ac);
209                                 shared_ptr<Resampler> re = resampler (ac, false);
210                                 if (re) {
211                                         shared_ptr<const AudioBuffers> b = re->flush ();
212                                         if (b->frames ()) {
213                                                 process_audio (earliest, b, ac->audio_length ());
214                                         }
215                                 }
216                         }
217                 }
218                 break;
219         }
220
221         if (_audio) {
222                 boost::optional<Time> audio_done_up_to;
223                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
224                         if ((*i)->decoder->done ()) {
225                                 continue;
226                         }
227
228                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
229                                 audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position);
230                         }
231                 }
232
233                 if (audio_done_up_to) {
234                         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to.get ());
235                         Audio (tb.audio, tb.time);
236                         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
237                 }
238         }
239                 
240         return false;
241 }
242
243 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
244 void
245 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame, Time extra)
246 {
247         /* Keep a note of what came in so that we can repeat it if required */
248         _last_incoming_video.weak_piece = weak_piece;
249         _last_incoming_video.image = image;
250         _last_incoming_video.eyes = eyes;
251         _last_incoming_video.same = same;
252         _last_incoming_video.frame = frame;
253         _last_incoming_video.extra = extra;
254         
255         shared_ptr<Piece> piece = weak_piece.lock ();
256         if (!piece) {
257                 return;
258         }
259
260         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
261         assert (content);
262
263         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
264         if (frc.skip && (frame % 2) == 1) {
265                 return;
266         }
267
268         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
269         if (content->trimmed (relative_time)) {
270                 return;
271         }
272
273         Time const time = content->position() + relative_time + extra - content->trim_start ();
274         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
275         libdcp::Size const image_size = fit_ratio_within (ratio, _video_container_size);
276         
277         shared_ptr<Image> work_image = image->crop_scale_window (content->crop(), image_size, _video_container_size, _film->scaler(), PIX_FMT_RGB24, false);
278
279         Position<int> const container_offset (
280                 (_video_container_size.width - image_size.width) / 2,
281                 (_video_container_size.height - image_size.width) / 2
282                 );
283
284         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
285                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position + container_offset);
286         }
287
288 #ifdef DCPOMATIC_DEBUG
289         _last_video = piece->content;
290 #endif
291
292         Video (work_image, eyes, content->colour_conversion(), same, time);
293
294         _last_emit_was_black = false;
295         _video_position = piece->video_position = (time + TIME_HZ / _film->video_frame_rate());
296
297         if (frc.repeat > 1 && !piece->repeating ()) {
298                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
299         }
300 }
301
302 void
303 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
304 {
305         shared_ptr<Piece> piece = weak_piece.lock ();
306         if (!piece) {
307                 return;
308         }
309
310         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
311         assert (content);
312
313         /* Gain */
314         if (content->audio_gain() != 0) {
315                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
316                 gain->apply_gain (content->audio_gain ());
317                 audio = gain;
318         }
319
320         /* Resample */
321         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
322                 shared_ptr<Resampler> r = resampler (content, true);
323                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
324                 audio = ro.first;
325                 frame = ro.second;
326         }
327         
328         Time const relative_time = _film->audio_frames_to_time (frame);
329
330         if (content->trimmed (relative_time)) {
331                 return;
332         }
333
334         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
335         
336         /* Remap channels */
337         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
338         dcp_mapped->make_silent ();
339         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
340         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
341                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
342                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
343                 }
344         }
345
346         audio = dcp_mapped;
347
348         /* We must cut off anything that comes before the start of all time */
349         if (time < 0) {
350                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
351                 if (frames >= audio->frames ()) {
352                         return;
353                 }
354
355                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
356                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
357
358                 audio = trimmed;
359                 time = 0;
360         }
361
362         _audio_merger.push (audio, time);
363         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
364 }
365
366 void
367 Player::flush ()
368 {
369         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
370         if (tb.audio) {
371                 Audio (tb.audio, tb.time);
372                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
373         }
374
375         while (_video_position < _audio_position) {
376                 emit_black ();
377         }
378
379         while (_audio_position < _video_position) {
380                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
381         }
382         
383 }
384
385 /** Seek so that the next pass() will yield (approximately) the requested frame.
386  *  Pass accurate = true to try harder to get close to the request.
387  *  @return true on error
388  */
389 void
390 Player::seek (Time t, bool accurate)
391 {
392         if (!_have_valid_pieces) {
393                 setup_pieces ();
394                 _have_valid_pieces = true;
395         }
396
397         if (_pieces.empty ()) {
398                 return;
399         }
400
401         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
402                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
403                 if (!vc) {
404                         continue;
405                 }
406
407                 /* s is the offset of t from the start position of this content */
408                 Time s = t - vc->position ();
409                 s = max (static_cast<Time> (0), s);
410                 s = min (vc->length_after_trim(), s);
411
412                 /* Hence set the piece positions to the `global' time */
413                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
414
415                 /* And seek the decoder */
416                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
417                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
418                         );
419
420                 (*i)->reset_repeat ();
421         }
422
423         _video_position = _audio_position = t;
424
425         /* XXX: don't seek audio because we don't need to... */
426 }
427
428 void
429 Player::setup_pieces ()
430 {
431         list<shared_ptr<Piece> > old_pieces = _pieces;
432
433         _pieces.clear ();
434
435         ContentList content = _playlist->content ();
436         sort (content.begin(), content.end(), ContentSorter ());
437
438         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
439
440                 shared_ptr<Piece> piece (new Piece (*i));
441
442                 /* XXX: into content? */
443
444                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
445                 if (fc) {
446                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
447                         
448                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
449                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
450                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
451
452                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
453                         piece->decoder = fd;
454                 }
455                 
456                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
457                 if (ic) {
458                         bool reusing = false;
459                         
460                         /* See if we can re-use an old ImageDecoder */
461                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
462                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
463                                 if (imd && imd->content() == ic) {
464                                         piece = *j;
465                                         reusing = true;
466                                 }
467                         }
468
469                         if (!reusing) {
470                                 shared_ptr<ImageDecoder> id (new ImageDecoder (_film, ic));
471                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
472                                 piece->decoder = id;
473                         }
474                 }
475
476                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
477                 if (sc) {
478                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
479                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
480
481                         piece->decoder = sd;
482                 }
483
484                 _pieces.push_back (piece);
485         }
486 }
487
488 void
489 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
490 {
491         shared_ptr<Content> c = w.lock ();
492         if (!c) {
493                 return;
494         }
495
496         if (
497                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
498                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
499                 ) {
500                 
501                 _have_valid_pieces = false;
502                 Changed (frequent);
503
504         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
505
506                 update_subtitle ();
507                 Changed (frequent);
508
509         } else if (
510                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
511                 property == VideoContentProperty::VIDEO_RATIO
512                 ) {
513                 
514                 Changed (frequent);
515
516         } else if (property == ContentProperty::PATH) {
517
518                 Changed (frequent);
519         }
520 }
521
522 void
523 Player::playlist_changed ()
524 {
525         _have_valid_pieces = false;
526         Changed (false);
527 }
528
529 void
530 Player::set_video_container_size (libdcp::Size s)
531 {
532         _video_container_size = s;
533         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
534         _black_frame->make_black ();
535 }
536
537 shared_ptr<Resampler>
538 Player::resampler (shared_ptr<AudioContent> c, bool create)
539 {
540         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
541         if (i != _resamplers.end ()) {
542                 return i->second;
543         }
544
545         if (!create) {
546                 return shared_ptr<Resampler> ();
547         }
548
549         _film->log()->log (
550                 String::compose (
551                         "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
552                         )
553                 );
554         
555         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
556         _resamplers[c] = r;
557         return r;
558 }
559
560 void
561 Player::emit_black ()
562 {
563 #ifdef DCPOMATIC_DEBUG
564         _last_video.reset ();
565 #endif
566
567         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
568         _video_position += _film->video_frames_to_time (1);
569         _last_emit_was_black = true;
570 }
571
572 void
573 Player::emit_silence (OutputAudioFrame most)
574 {
575         if (most == 0) {
576                 return;
577         }
578         
579         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
580         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
581         silence->make_silent ();
582         Audio (silence, _audio_position);
583         _audio_position += _film->audio_frames_to_time (N);
584 }
585
586 void
587 Player::film_changed (Film::Property p)
588 {
589         /* Here we should notice Film properties that affect our output, and
590            alert listeners that our output now would be different to how it was
591            last time we were run.
592         */
593
594         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
595                 Changed (false);
596         }
597 }
598
599 void
600 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
601 {
602         _in_subtitle.piece = weak_piece;
603         _in_subtitle.image = image;
604         _in_subtitle.rect = rect;
605         _in_subtitle.from = from;
606         _in_subtitle.to = to;
607
608         update_subtitle ();
609 }
610
611 void
612 Player::update_subtitle ()
613 {
614         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
615         if (!piece) {
616                 return;
617         }
618
619         if (!_in_subtitle.image) {
620                 _out_subtitle.image.reset ();
621                 return;
622         }
623
624         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
625         assert (sc);
626
627         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
628         libdcp::Size scaled_size;
629
630         in_rect.y += sc->subtitle_offset ();
631
632         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
633         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
634         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
635
636         /* Then we need a corrective translation, consisting of two parts:
637          *
638          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
639          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
640          *
641          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
642          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
643          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
644          *
645          * Combining these two translations gives these expressions.
646          */
647         
648         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
649         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
650         
651         _out_subtitle.image = _in_subtitle.image->scale (
652                 scaled_size,
653                 Scaler::from_id ("bicubic"),
654                 _in_subtitle.image->pixel_format (),
655                 true
656                 );
657         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
658         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
659 }
660
661 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
662  *  @return false if this could not be done.
663  */
664 bool
665 Player::repeat_last_video ()
666 {
667         if (!_last_incoming_video.image) {
668                 return false;
669         }
670
671         process_video (
672                 _last_incoming_video.weak_piece,
673                 _last_incoming_video.image,
674                 _last_incoming_video.eyes,
675                 _last_incoming_video.same,
676                 _last_incoming_video.frame,
677                 _last_incoming_video.extra
678                 );
679
680         return true;
681 }