Fix build.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "still_image_decoder.h"
26 #include "still_image_content.h"
27 #include "moving_image_decoder.h"
28 #include "moving_image_content.h"
29 #include "sndfile_decoder.h"
30 #include "sndfile_content.h"
31 #include "subtitle_content.h"
32 #include "playlist.h"
33 #include "job.h"
34 #include "image.h"
35 #include "ratio.h"
36 #include "resampler.h"
37 #include "log.h"
38 #include "scaler.h"
39
40 using std::list;
41 using std::cout;
42 using std::min;
43 using std::max;
44 using std::vector;
45 using std::pair;
46 using std::map;
47 using boost::shared_ptr;
48 using boost::weak_ptr;
49 using boost::dynamic_pointer_cast;
50
51 //#define DEBUG_PLAYER 1
52
53 class Piece
54 {
55 public:
56         Piece (shared_ptr<Content> c)
57                 : content (c)
58                 , video_position (c->position ())
59                 , audio_position (c->position ())
60         {}
61         
62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
63                 : content (c)
64                 , decoder (d)
65                 , video_position (c->position ())
66                 , audio_position (c->position ())
67         {}
68         
69         shared_ptr<Content> content;
70         shared_ptr<Decoder> decoder;
71         Time video_position;
72         Time audio_position;
73 };
74
75 #ifdef DEBUG_PLAYER
76 std::ostream& operator<<(std::ostream& s, Piece const & p)
77 {
78         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
79                 s << "\tffmpeg     ";
80         } else if (dynamic_pointer_cast<StillImageContent> (p.content)) {
81                 s << "\tstill image";
82         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
83                 s << "\tsndfile    ";
84         }
85         
86         s << " at " << p.content->position() << " until " << p.content->end();
87         
88         return s;
89 }
90 #endif  
91
92 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
93         : _film (f)
94         , _playlist (p)
95         , _video (true)
96         , _audio (true)
97         , _have_valid_pieces (false)
98         , _video_position (0)
99         , _audio_position (0)
100         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
101         , _last_emit_was_black (false)
102 {
103         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
104         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
105         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
106         set_video_container_size (_film->container()->size (_film->full_frame ()));
107 }
108
109 void
110 Player::disable_video ()
111 {
112         _video = false;
113 }
114
115 void
116 Player::disable_audio ()
117 {
118         _audio = false;
119 }
120
121 bool
122 Player::pass ()
123 {
124         if (!_have_valid_pieces) {
125                 setup_pieces ();
126                 _have_valid_pieces = true;
127         }
128
129 #ifdef DEBUG_PLAYER
130         cout << "= PASS\n";
131 #endif  
132
133         Time earliest_t = TIME_MAX;
134         shared_ptr<Piece> earliest;
135         enum {
136                 VIDEO,
137                 AUDIO
138         } type = VIDEO;
139
140         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
141                 if ((*i)->decoder->done ()) {
142                         continue;
143                 }
144
145                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
146                         if ((*i)->video_position < earliest_t) {
147                                 earliest_t = (*i)->video_position;
148                                 earliest = *i;
149                                 type = VIDEO;
150                         }
151                 }
152
153                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
154                         if ((*i)->audio_position < earliest_t) {
155                                 earliest_t = (*i)->audio_position;
156                                 earliest = *i;
157                                 type = AUDIO;
158                         }
159                 }
160         }
161
162         if (!earliest) {
163 #ifdef DEBUG_PLAYER
164                 cout << "no earliest piece.\n";
165 #endif          
166                 
167                 flush ();
168                 return true;
169         }
170
171         switch (type) {
172         case VIDEO:
173                 if (earliest_t > _video_position) {
174 #ifdef DEBUG_PLAYER
175                         cout << "no video here; emitting black frame (earliest=" << earliest_t << ", video_position=" << _video_position << ").\n";
176 #endif
177                         emit_black ();
178                 } else {
179 #ifdef DEBUG_PLAYER
180                         cout << "Pass video " << *earliest << "\n";
181 #endif                  
182                         earliest->decoder->pass ();
183                 }
184                 break;
185
186         case AUDIO:
187                 if (earliest_t > _audio_position) {
188 #ifdef DEBUG_PLAYER
189                         cout << "no audio here (none until " << earliest_t << "); emitting silence.\n";
190 #endif
191                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
192                 } else {
193 #ifdef DEBUG_PLAYER
194                         cout << "Pass audio " << *earliest << "\n";
195 #endif
196                         earliest->decoder->pass ();
197
198                         if (earliest->decoder->done()) {
199                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
200                                 assert (ac);
201                                 shared_ptr<Resampler> re = resampler (ac, false);
202                                 if (re) {
203                                         shared_ptr<const AudioBuffers> b = re->flush ();
204                                         if (b->frames ()) {
205                                                 process_audio (earliest, b, ac->audio_length ());
206                                         }
207                                 }
208                         }
209                 }
210                 break;
211         }
212
213         if (_audio) {
214                 Time audio_done_up_to = TIME_MAX;
215                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
216                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
217                                 audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
218                         }
219                 }
220
221                 TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
222                 Audio (tb.audio, tb.time);
223                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
224         }
225                 
226 #ifdef DEBUG_PLAYER
227         cout << "\tpost pass _video_position=" << _video_position << " _audio_position=" << _audio_position << "\n";
228 #endif  
229
230         return false;
231 }
232
233 void
234 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
235 {
236         shared_ptr<Piece> piece = weak_piece.lock ();
237         if (!piece) {
238                 return;
239         }
240
241         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
242         assert (content);
243
244         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
245         if (frc.skip && (frame % 2) == 1) {
246                 return;
247         }
248
249         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
250         if (content->trimmed (relative_time)) {
251                 return;
252         }
253
254         /* Convert to RGB first, as FFmpeg doesn't seem to like handling YUV images with odd widths */
255         shared_ptr<Image> work_image = image->scale (image->size (), _film->scaler(), PIX_FMT_RGB24, true);
256
257         work_image = work_image->crop (content->crop(), true);
258
259         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
260         
261         work_image = work_image->scale (image_size, _film->scaler(), PIX_FMT_RGB24, true);
262
263         Time time = content->position() + relative_time - content->trim_start ();
264             
265         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
266                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
267         }
268
269         if (image_size != _video_container_size) {
270                 assert (image_size.width <= _video_container_size.width);
271                 assert (image_size.height <= _video_container_size.height);
272                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
273                 im->make_black ();
274                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
275                 work_image = im;
276         }
277
278 #ifdef DCPOMATIC_DEBUG
279         _last_video = piece->content;
280 #endif
281
282         Video (work_image, eyes, content->colour_conversion(), same, time);
283         time += TIME_HZ / _film->video_frame_rate();
284
285         if (frc.repeat) {
286                 Video (work_image, eyes, content->colour_conversion(), true, time);
287                 time += TIME_HZ / _film->video_frame_rate();
288         }
289
290         _last_emit_was_black = false;
291
292         _video_position = piece->video_position = time;
293 }
294
295 void
296 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
297 {
298         shared_ptr<Piece> piece = weak_piece.lock ();
299         if (!piece) {
300                 return;
301         }
302
303         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
304         assert (content);
305
306         /* Gain */
307         if (content->audio_gain() != 0) {
308                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
309                 gain->apply_gain (content->audio_gain ());
310                 audio = gain;
311         }
312
313         /* Resample */
314         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
315                 shared_ptr<Resampler> r = resampler (content, true);
316                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
317                 audio = ro.first;
318                 frame = ro.second;
319         }
320         
321         Time const relative_time = _film->audio_frames_to_time (frame);
322
323         if (content->trimmed (relative_time)) {
324                 return;
325         }
326
327         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time;
328         
329         /* Remap channels */
330         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
331         dcp_mapped->make_silent ();
332         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
333         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
334                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
335                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
336                 }
337         }
338
339         audio = dcp_mapped;
340
341         /* We must cut off anything that comes before the start of all time */
342         if (time < 0) {
343                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
344                 if (frames >= audio->frames ()) {
345                         return;
346                 }
347
348                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
349                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
350
351                 audio = trimmed;
352                 time = 0;
353         }
354
355         _audio_merger.push (audio, time);
356         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
357 }
358
359 void
360 Player::flush ()
361 {
362         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
363         if (tb.audio) {
364                 Audio (tb.audio, tb.time);
365                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
366         }
367
368         while (_video_position < _audio_position) {
369                 emit_black ();
370         }
371
372         while (_audio_position < _video_position) {
373                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
374         }
375         
376 }
377
378 /** Seek so that the next pass() will yield (approximately) the requested frame.
379  *  Pass accurate = true to try harder to get close to the request.
380  *  @return true on error
381  */
382 void
383 Player::seek (Time t, bool accurate)
384 {
385         if (!_have_valid_pieces) {
386                 setup_pieces ();
387                 _have_valid_pieces = true;
388         }
389
390         if (_pieces.empty ()) {
391                 return;
392         }
393
394         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
395                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
396                 if (!vc) {
397                         continue;
398                 }
399                 
400                 Time s = t - vc->position ();
401                 s = max (static_cast<Time> (0), s);
402                 s = min (vc->length_after_trim(), s);
403
404                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
405
406                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
407                 /* Here we are converting from time (in the DCP) to a frame number in the content.
408                    Hence we need to use the DCP's frame rate and the double/skip correction, not
409                    the source's rate.
410                 */
411                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
412                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
413         }
414
415         _video_position = _audio_position = t;
416         
417         /* XXX: don't seek audio because we don't need to... */
418 }
419
420 void
421 Player::setup_pieces ()
422 {
423         list<shared_ptr<Piece> > old_pieces = _pieces;
424
425         _pieces.clear ();
426
427         ContentList content = _playlist->content ();
428         sort (content.begin(), content.end(), ContentSorter ());
429
430         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
431
432                 shared_ptr<Piece> piece (new Piece (*i));
433
434                 /* XXX: into content? */
435
436                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
437                 if (fc) {
438                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
439                         
440                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
441                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
442                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
443
444                         piece->decoder = fd;
445                 }
446                 
447                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
448                 if (ic) {
449                         shared_ptr<StillImageDecoder> id;
450                         
451                         /* See if we can re-use an old StillImageDecoder */
452                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
453                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
454                                 if (imd && imd->content() == ic) {
455                                         id = imd;
456                                 }
457                         }
458
459                         if (!id) {
460                                 id.reset (new StillImageDecoder (_film, ic));
461                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
462                         }
463
464                         piece->decoder = id;
465                 }
466
467                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
468                 if (mc) {
469                         shared_ptr<MovingImageDecoder> md;
470
471                         if (!md) {
472                                 md.reset (new MovingImageDecoder (_film, mc));
473                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
474                         }
475
476                         piece->decoder = md;
477                 }
478
479                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
480                 if (sc) {
481                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
482                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
483
484                         piece->decoder = sd;
485                 }
486
487                 _pieces.push_back (piece);
488         }
489
490 #ifdef DEBUG_PLAYER
491         cout << "=== Player setup:\n";
492         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
493                 cout << *(i->get()) << "\n";
494         }
495 #endif  
496 }
497
498 void
499 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
500 {
501         shared_ptr<Content> c = w.lock ();
502         if (!c) {
503                 return;
504         }
505
506         if (
507                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
508                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
509                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO
510                 ) {
511                 
512                 _have_valid_pieces = false;
513                 Changed (frequent);
514
515         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
516                 update_subtitle ();
517                 Changed (frequent);
518         } else if (property == VideoContentProperty::VIDEO_FRAME_TYPE) {
519                 Changed (frequent);
520         }
521 }
522
523 void
524 Player::playlist_changed ()
525 {
526         _have_valid_pieces = false;
527         Changed (false);
528 }
529
530 void
531 Player::set_video_container_size (libdcp::Size s)
532 {
533         _video_container_size = s;
534         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
535         _black_frame->make_black ();
536 }
537
538 shared_ptr<Resampler>
539 Player::resampler (shared_ptr<AudioContent> c, bool create)
540 {
541         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
542         if (i != _resamplers.end ()) {
543                 return i->second;
544         }
545
546         if (!create) {
547                 return shared_ptr<Resampler> ();
548         }
549         
550         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
551         _resamplers[c] = r;
552         return r;
553 }
554
555 void
556 Player::emit_black ()
557 {
558 #ifdef DCPOMATIC_DEBUG
559         _last_video.reset ();
560 #endif
561         
562         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
563         _video_position += _film->video_frames_to_time (1);
564         _last_emit_was_black = true;
565 }
566
567 void
568 Player::emit_silence (OutputAudioFrame most)
569 {
570         if (most == 0) {
571                 return;
572         }
573         
574         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
575         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
576         silence->make_silent ();
577         Audio (silence, _audio_position);
578         _audio_position += _film->audio_frames_to_time (N);
579 }
580
581 void
582 Player::film_changed (Film::Property p)
583 {
584         /* Here we should notice Film properties that affect our output, and
585            alert listeners that our output now would be different to how it was
586            last time we were run.
587         */
588
589         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
590                 Changed (false);
591         }
592 }
593
594 void
595 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
596 {
597         _in_subtitle.piece = weak_piece;
598         _in_subtitle.image = image;
599         _in_subtitle.rect = rect;
600         _in_subtitle.from = from;
601         _in_subtitle.to = to;
602
603         update_subtitle ();
604 }
605
606 void
607 Player::update_subtitle ()
608 {
609         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
610         if (!piece) {
611                 return;
612         }
613
614         if (!_in_subtitle.image) {
615                 _out_subtitle.image.reset ();
616                 return;
617         }
618
619         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
620         assert (sc);
621
622         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
623         libdcp::Size scaled_size;
624
625         in_rect.y += sc->subtitle_offset ();
626
627         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
628         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
629         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
630
631         /* Then we need a corrective translation, consisting of two parts:
632          *
633          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
634          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
635          *
636          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
637          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
638          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
639          *
640          * Combining these two translations gives these expressions.
641          */
642         
643         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
644         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
645         
646         _out_subtitle.image = _in_subtitle.image->scale (
647                 scaled_size,
648                 Scaler::from_id ("bicubic"),
649                 _in_subtitle.image->pixel_format (),
650                 true
651                 );
652         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
653         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
654 }