Merge 1.0.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "still_image_decoder.h"
26 #include "still_image_content.h"
27 #include "moving_image_decoder.h"
28 #include "moving_image_content.h"
29 #include "sndfile_decoder.h"
30 #include "sndfile_content.h"
31 #include "subtitle_content.h"
32 #include "playlist.h"
33 #include "job.h"
34 #include "image.h"
35 #include "ratio.h"
36 #include "resampler.h"
37 #include "log.h"
38 #include "scaler.h"
39
40 using std::list;
41 using std::cout;
42 using std::min;
43 using std::max;
44 using std::vector;
45 using std::pair;
46 using std::map;
47 using boost::shared_ptr;
48 using boost::weak_ptr;
49 using boost::dynamic_pointer_cast;
50
51 class Piece
52 {
53 public:
54         Piece (shared_ptr<Content> c)
55                 : content (c)
56                 , video_position (c->position ())
57                 , audio_position (c->position ())
58                 , repeat_to_do (0)
59                 , repeat_done (0)
60         {}
61         
62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
63                 : content (c)
64                 , decoder (d)
65                 , video_position (c->position ())
66                 , audio_position (c->position ())
67         {}
68
69         void set_repeat (IncomingVideo video, int num)
70         {
71                 cout << "Set repeat " << num << "\n";
72                 repeat_video = video;
73                 repeat_to_do = num;
74                 repeat_done = 0;
75         }
76
77         void reset_repeat ()
78         {
79                 repeat_video.image.reset ();
80                 repeat_to_do = 0;
81                 repeat_done = 0;
82         }
83
84         bool repeating () const
85         {
86                 return repeat_done != repeat_to_do;
87         }
88
89         void repeat (Player* player)
90         {
91                 cout << "repeating; " << repeat_done << "\n";
92                 player->process_video (
93                         repeat_video.weak_piece,
94                         repeat_video.image,
95                         repeat_video.eyes,
96                         repeat_video.same,
97                         repeat_video.frame,
98                         (repeat_done + 1) * (TIME_HZ / player->_film->video_frame_rate ())
99                         );
100
101                 ++repeat_done;
102         }
103         
104         shared_ptr<Content> content;
105         shared_ptr<Decoder> decoder;
106         Time video_position;
107         Time audio_position;
108
109         IncomingVideo repeat_video;
110         int repeat_to_do;
111         int repeat_done;
112 };
113
114 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
115         : _film (f)
116         , _playlist (p)
117         , _video (true)
118         , _audio (true)
119         , _have_valid_pieces (false)
120         , _video_position (0)
121         , _audio_position (0)
122         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
123         , _last_emit_was_black (false)
124 {
125         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
126         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
127         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
128         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
129 }
130
131 void
132 Player::disable_video ()
133 {
134         _video = false;
135 }
136
137 void
138 Player::disable_audio ()
139 {
140         _audio = false;
141 }
142
143 bool
144 Player::pass ()
145 {
146         if (!_have_valid_pieces) {
147                 setup_pieces ();
148                 _have_valid_pieces = true;
149         }
150
151         Time earliest_t = TIME_MAX;
152         shared_ptr<Piece> earliest;
153         enum {
154                 VIDEO,
155                 AUDIO
156         } type = VIDEO;
157
158         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
159                 if ((*i)->decoder->done ()) {
160                         cout << "Scan: done.\n";
161                         continue;
162                 }
163
164                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
165                         if ((*i)->video_position < earliest_t) {
166                                 earliest_t = (*i)->video_position;
167                                 earliest = *i;
168                                 type = VIDEO;
169                         }
170                 }
171
172                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
173                         if ((*i)->audio_position < earliest_t) {
174                                 earliest_t = (*i)->audio_position;
175                                 earliest = *i;
176                                 type = AUDIO;
177                         }
178                 }
179         }
180
181         if (!earliest) {
182                 cout << "No earliest: out.\n";
183                 flush ();
184                 return true;
185         }
186
187         cout << "Earliest: " << earliest_t << "\n";
188
189         switch (type) {
190         case VIDEO:
191                 cout << "VIDEO.\n";
192                 if (earliest_t > _video_position) {
193                         emit_black ();
194                 } else {
195                         if (earliest->repeating ()) {
196                                 cout << "-repeating.\n";
197                                 earliest->repeat (this);
198                         } else {
199                                 cout << "-passing.\n";
200                                 earliest->decoder->pass ();
201                         }
202                 }
203                 break;
204
205         case AUDIO:
206                 cout << "SOUND.\n";
207                 if (earliest_t > _audio_position) {
208                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
209                 } else {
210                         earliest->decoder->pass ();
211
212                         if (earliest->decoder->done()) {
213                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
214                                 assert (ac);
215                                 shared_ptr<Resampler> re = resampler (ac, false);
216                                 if (re) {
217                                         shared_ptr<const AudioBuffers> b = re->flush ();
218                                         if (b->frames ()) {
219                                                 process_audio (earliest, b, ac->audio_length ());
220                                         }
221                                 }
222                         }
223                 }
224                 break;
225         }
226
227         if (_audio) {
228                 Time audio_done_up_to = TIME_MAX;
229                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
230                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
231                                 audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
232                         }
233                 }
234
235                 TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
236                 Audio (tb.audio, tb.time);
237                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
238         }
239                 
240         return false;
241 }
242
243 void
244 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame, Time extra)
245 {
246         cout << "PLAYER RECEIVES A VIDEO FRAME, extra " << extra << "\n";
247         
248         /* Keep a note of what came in so that we can repeat it if required */
249         _last_incoming_video.weak_piece = weak_piece;
250         _last_incoming_video.image = image;
251         _last_incoming_video.eyes = eyes;
252         _last_incoming_video.same = same;
253         _last_incoming_video.frame = frame;
254         
255         shared_ptr<Piece> piece = weak_piece.lock ();
256         if (!piece) {
257                 return;
258         }
259
260         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
261         assert (content);
262
263         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
264         if (frc.skip && (frame % 2) == 1) {
265                 return;
266         }
267
268         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
269         if (content->trimmed (relative_time)) {
270                 return;
271         }
272
273         /* Convert to RGB first, as FFmpeg doesn't seem to like handling YUV images with odd widths */
274         shared_ptr<Image> work_image = image->scale (image->size (), _film->scaler(), PIX_FMT_RGB24, true);
275
276         work_image = work_image->crop (content->crop(), true);
277
278         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
279         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
280         
281         work_image = work_image->scale (image_size, _film->scaler(), PIX_FMT_RGB24, true);
282
283         Time time = content->position() + relative_time + extra - content->trim_start ();
284             
285         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
286                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
287         }
288
289         if (image_size != _video_container_size) {
290                 assert (image_size.width <= _video_container_size.width);
291                 assert (image_size.height <= _video_container_size.height);
292                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
293                 im->make_black ();
294                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
295                 work_image = im;
296         }
297
298 #ifdef DCPOMATIC_DEBUG
299         _last_video = piece->content;
300 #endif
301
302         Video (work_image, eyes, content->colour_conversion(), same, time);
303
304         time += TIME_HZ / _film->video_frame_rate();
305         _last_emit_was_black = false;
306         _video_position = piece->video_position = time;
307
308         cout << "frc.repeat=" << frc.repeat << "; vp now " << _video_position << "\n";
309
310         if (frc.repeat > 1 && !piece->repeating ()) {
311                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
312         }
313 }
314
315 void
316 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
317 {
318         shared_ptr<Piece> piece = weak_piece.lock ();
319         if (!piece) {
320                 return;
321         }
322
323         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
324         assert (content);
325
326         /* Gain */
327         if (content->audio_gain() != 0) {
328                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
329                 gain->apply_gain (content->audio_gain ());
330                 audio = gain;
331         }
332
333         /* Resample */
334         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
335                 shared_ptr<Resampler> r = resampler (content, true);
336                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
337                 audio = ro.first;
338                 frame = ro.second;
339         }
340         
341         Time const relative_time = _film->audio_frames_to_time (frame);
342
343         if (content->trimmed (relative_time)) {
344                 return;
345         }
346
347         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time;
348         
349         /* Remap channels */
350         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
351         dcp_mapped->make_silent ();
352         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
353         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
354                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
355                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
356                 }
357         }
358
359         audio = dcp_mapped;
360
361         /* We must cut off anything that comes before the start of all time */
362         if (time < 0) {
363                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
364                 if (frames >= audio->frames ()) {
365                         return;
366                 }
367
368                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
369                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
370
371                 audio = trimmed;
372                 time = 0;
373         }
374
375         _audio_merger.push (audio, time);
376         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
377 }
378
379 void
380 Player::flush ()
381 {
382         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
383         if (tb.audio) {
384                 Audio (tb.audio, tb.time);
385                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
386         }
387
388         while (_video_position < _audio_position) {
389                 emit_black ();
390         }
391
392         while (_audio_position < _video_position) {
393                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
394         }
395         
396 }
397
398 /** Seek so that the next pass() will yield (approximately) the requested frame.
399  *  Pass accurate = true to try harder to get close to the request.
400  *  @return true on error
401  */
402 void
403 Player::seek (Time t, bool accurate)
404 {
405         if (!_have_valid_pieces) {
406                 setup_pieces ();
407                 _have_valid_pieces = true;
408         }
409
410         if (_pieces.empty ()) {
411                 return;
412         }
413
414         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
415                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
416                 if (!vc) {
417                         continue;
418                 }
419                 
420                 Time s = t - vc->position ();
421                 s = max (static_cast<Time> (0), s);
422                 s = min (vc->length_after_trim(), s);
423
424                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
425
426                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
427                 /* Here we are converting from time (in the DCP) to a frame number in the content.
428                    Hence we need to use the DCP's frame rate and the double/skip correction, not
429                    the source's rate.
430                 */
431                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
432                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
433
434                 (*i)->reset_repeat ();
435         }
436
437         _video_position = _audio_position = t;
438
439         /* XXX: don't seek audio because we don't need to... */
440 }
441
442 void
443 Player::setup_pieces ()
444 {
445         list<shared_ptr<Piece> > old_pieces = _pieces;
446
447         _pieces.clear ();
448
449         ContentList content = _playlist->content ();
450         sort (content.begin(), content.end(), ContentSorter ());
451
452         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
453
454                 shared_ptr<Piece> piece (new Piece (*i));
455
456                 /* XXX: into content? */
457
458                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
459                 if (fc) {
460                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
461                         
462                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4, 0));
463                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
464                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
465
466                         piece->decoder = fd;
467                 }
468                 
469                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
470                 if (ic) {
471                         shared_ptr<StillImageDecoder> id;
472                         
473                         /* See if we can re-use an old StillImageDecoder */
474                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
475                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
476                                 if (imd && imd->content() == ic) {
477                                         id = imd;
478                                 }
479                         }
480
481                         if (!id) {
482                                 id.reset (new StillImageDecoder (_film, ic));
483                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4, 0));
484                         }
485
486                         piece->decoder = id;
487                 }
488
489                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
490                 if (mc) {
491                         shared_ptr<MovingImageDecoder> md;
492
493                         if (!md) {
494                                 md.reset (new MovingImageDecoder (_film, mc));
495                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4, 0));
496                         }
497
498                         piece->decoder = md;
499                 }
500
501                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
502                 if (sc) {
503                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
504                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
505
506                         piece->decoder = sd;
507                 }
508
509                 _pieces.push_back (piece);
510         }
511 }
512
513 void
514 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
515 {
516         shared_ptr<Content> c = w.lock ();
517         if (!c) {
518                 return;
519         }
520
521         if (
522                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
523                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
524                 ) {
525                 
526                 _have_valid_pieces = false;
527                 Changed (frequent);
528
529         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
530
531                 update_subtitle ();
532                 Changed (frequent);
533
534         } else if (
535                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
536                 property == VideoContentProperty::VIDEO_RATIO
537                 ) {
538                 
539                 Changed (frequent);
540
541         } else if (property == ContentProperty::PATH) {
542
543                 Changed (frequent);
544         }
545 }
546
547 void
548 Player::playlist_changed ()
549 {
550         _have_valid_pieces = false;
551         Changed (false);
552 }
553
554 void
555 Player::set_video_container_size (libdcp::Size s)
556 {
557         _video_container_size = s;
558         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
559         _black_frame->make_black ();
560 }
561
562 shared_ptr<Resampler>
563 Player::resampler (shared_ptr<AudioContent> c, bool create)
564 {
565         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
566         if (i != _resamplers.end ()) {
567                 return i->second;
568         }
569
570         if (!create) {
571                 return shared_ptr<Resampler> ();
572         }
573         
574         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
575         _resamplers[c] = r;
576         return r;
577 }
578
579 void
580 Player::emit_black ()
581 {
582 #ifdef DCPOMATIC_DEBUG
583         _last_video.reset ();
584 #endif
585         
586         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
587         _video_position += _film->video_frames_to_time (1);
588         _last_emit_was_black = true;
589 }
590
591 void
592 Player::emit_silence (OutputAudioFrame most)
593 {
594         if (most == 0) {
595                 return;
596         }
597         
598         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
599         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
600         silence->make_silent ();
601         Audio (silence, _audio_position);
602         _audio_position += _film->audio_frames_to_time (N);
603 }
604
605 void
606 Player::film_changed (Film::Property p)
607 {
608         /* Here we should notice Film properties that affect our output, and
609            alert listeners that our output now would be different to how it was
610            last time we were run.
611         */
612
613         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
614                 Changed (false);
615         }
616 }
617
618 void
619 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
620 {
621         _in_subtitle.piece = weak_piece;
622         _in_subtitle.image = image;
623         _in_subtitle.rect = rect;
624         _in_subtitle.from = from;
625         _in_subtitle.to = to;
626
627         update_subtitle ();
628 }
629
630 void
631 Player::update_subtitle ()
632 {
633         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
634         if (!piece) {
635                 return;
636         }
637
638         if (!_in_subtitle.image) {
639                 _out_subtitle.image.reset ();
640                 return;
641         }
642
643         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
644         assert (sc);
645
646         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
647         libdcp::Size scaled_size;
648
649         in_rect.y += sc->subtitle_offset ();
650
651         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
652         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
653         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
654
655         /* Then we need a corrective translation, consisting of two parts:
656          *
657          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
658          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
659          *
660          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
661          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
662          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
663          *
664          * Combining these two translations gives these expressions.
665          */
666         
667         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
668         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
669         
670         _out_subtitle.image = _in_subtitle.image->scale (
671                 scaled_size,
672                 Scaler::from_id ("bicubic"),
673                 _in_subtitle.image->pixel_format (),
674                 true
675                 );
676         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
677         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
678 }
679
680 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
681  *  @return false if this could not be done.
682  */
683 bool
684 Player::repeat_last_video ()
685 {
686         if (!_last_incoming_video.image) {
687                 return false;
688         }
689
690         process_video (
691                 _last_incoming_video.weak_piece,
692                 _last_incoming_video.image,
693                 _last_incoming_video.eyes,
694                 _last_incoming_video.same,
695                 _last_incoming_video.frame,
696                 0
697                 );
698
699         return true;
700 }