Allow no-stretch scaling of video content.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include <stdint.h>
21 #include "player.h"
22 #include "film.h"
23 #include "ffmpeg_decoder.h"
24 #include "ffmpeg_content.h"
25 #include "still_image_decoder.h"
26 #include "still_image_content.h"
27 #include "moving_image_decoder.h"
28 #include "moving_image_content.h"
29 #include "sndfile_decoder.h"
30 #include "sndfile_content.h"
31 #include "subtitle_content.h"
32 #include "playlist.h"
33 #include "job.h"
34 #include "image.h"
35 #include "ratio.h"
36 #include "resampler.h"
37 #include "log.h"
38 #include "scaler.h"
39
40 using std::list;
41 using std::cout;
42 using std::min;
43 using std::max;
44 using std::vector;
45 using std::pair;
46 using std::map;
47 using boost::shared_ptr;
48 using boost::weak_ptr;
49 using boost::dynamic_pointer_cast;
50
51 //#define DEBUG_PLAYER 1
52
53 class Piece
54 {
55 public:
56         Piece (shared_ptr<Content> c)
57                 : content (c)
58                 , video_position (c->position ())
59                 , audio_position (c->position ())
60         {}
61         
62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
63                 : content (c)
64                 , decoder (d)
65                 , video_position (c->position ())
66                 , audio_position (c->position ())
67         {}
68         
69         shared_ptr<Content> content;
70         shared_ptr<Decoder> decoder;
71         Time video_position;
72         Time audio_position;
73 };
74
75 #ifdef DEBUG_PLAYER
76 std::ostream& operator<<(std::ostream& s, Piece const & p)
77 {
78         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
79                 s << "\tffmpeg     ";
80         } else if (dynamic_pointer_cast<StillImageContent> (p.content)) {
81                 s << "\tstill image";
82         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
83                 s << "\tsndfile    ";
84         }
85         
86         s << " at " << p.content->position() << " until " << p.content->end();
87         
88         return s;
89 }
90 #endif  
91
92 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
93         : _film (f)
94         , _playlist (p)
95         , _video (true)
96         , _audio (true)
97         , _have_valid_pieces (false)
98         , _video_position (0)
99         , _audio_position (0)
100         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
101         , _last_emit_was_black (false)
102 {
103         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
104         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
105         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
106         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
107 }
108
109 void
110 Player::disable_video ()
111 {
112         _video = false;
113 }
114
115 void
116 Player::disable_audio ()
117 {
118         _audio = false;
119 }
120
121 bool
122 Player::pass ()
123 {
124         if (!_have_valid_pieces) {
125                 setup_pieces ();
126                 _have_valid_pieces = true;
127         }
128
129 #ifdef DEBUG_PLAYER
130         cout << "= PASS\n";
131 #endif  
132
133         Time earliest_t = TIME_MAX;
134         shared_ptr<Piece> earliest;
135         enum {
136                 VIDEO,
137                 AUDIO
138         } type = VIDEO;
139
140         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
141                 if ((*i)->decoder->done ()) {
142                         continue;
143                 }
144
145                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
146                         if ((*i)->video_position < earliest_t) {
147                                 earliest_t = (*i)->video_position;
148                                 earliest = *i;
149                                 type = VIDEO;
150                         }
151                 }
152
153                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
154                         if ((*i)->audio_position < earliest_t) {
155                                 earliest_t = (*i)->audio_position;
156                                 earliest = *i;
157                                 type = AUDIO;
158                         }
159                 }
160         }
161
162         if (!earliest) {
163 #ifdef DEBUG_PLAYER
164                 cout << "no earliest piece.\n";
165 #endif          
166                 
167                 flush ();
168                 return true;
169         }
170
171         switch (type) {
172         case VIDEO:
173                 if (earliest_t > _video_position) {
174 #ifdef DEBUG_PLAYER
175                         cout << "no video here; emitting black frame (earliest=" << earliest_t << ", video_position=" << _video_position << ").\n";
176 #endif
177                         emit_black ();
178                 } else {
179 #ifdef DEBUG_PLAYER
180                         cout << "Pass video " << *earliest << "\n";
181 #endif                  
182                         earliest->decoder->pass ();
183                 }
184                 break;
185
186         case AUDIO:
187                 if (earliest_t > _audio_position) {
188 #ifdef DEBUG_PLAYER
189                         cout << "no audio here (none until " << earliest_t << "); emitting silence.\n";
190 #endif
191                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
192                 } else {
193 #ifdef DEBUG_PLAYER
194                         cout << "Pass audio " << *earliest << "\n";
195 #endif
196                         earliest->decoder->pass ();
197
198                         if (earliest->decoder->done()) {
199                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
200                                 assert (ac);
201                                 shared_ptr<Resampler> re = resampler (ac, false);
202                                 if (re) {
203                                         shared_ptr<const AudioBuffers> b = re->flush ();
204                                         if (b->frames ()) {
205                                                 process_audio (earliest, b, ac->audio_length ());
206                                         }
207                                 }
208                         }
209                 }
210                 break;
211         }
212
213         if (_audio) {
214                 Time audio_done_up_to = TIME_MAX;
215                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
216                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
217                                 audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
218                         }
219                 }
220
221                 TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
222                 Audio (tb.audio, tb.time);
223                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
224         }
225                 
226 #ifdef DEBUG_PLAYER
227         cout << "\tpost pass _video_position=" << _video_position << " _audio_position=" << _audio_position << "\n";
228 #endif  
229
230         return false;
231 }
232
233 void
234 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
235 {
236         shared_ptr<Piece> piece = weak_piece.lock ();
237         if (!piece) {
238                 return;
239         }
240
241         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
242         assert (content);
243
244         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
245         if (frc.skip && (frame % 2) == 1) {
246                 return;
247         }
248
249         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
250         if (content->trimmed (relative_time)) {
251                 return;
252         }
253
254         /* Convert to RGB first, as FFmpeg doesn't seem to like handling YUV images with odd widths */
255         shared_ptr<Image> work_image = image->scale (image->size (), _film->scaler(), PIX_FMT_RGB24, true);
256
257         work_image = work_image->crop (content->crop(), true);
258
259         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
260         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
261         
262         work_image = work_image->scale (image_size, _film->scaler(), PIX_FMT_RGB24, true);
263
264         Time time = content->position() + relative_time - content->trim_start ();
265             
266         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
267                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
268         }
269
270         if (image_size != _video_container_size) {
271                 assert (image_size.width <= _video_container_size.width);
272                 assert (image_size.height <= _video_container_size.height);
273                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
274                 im->make_black ();
275                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
276                 work_image = im;
277         }
278
279 #ifdef DCPOMATIC_DEBUG
280         _last_video = piece->content;
281 #endif
282
283         Video (work_image, eyes, content->colour_conversion(), same, time);
284         time += TIME_HZ / _film->video_frame_rate();
285
286         if (frc.repeat) {
287                 Video (work_image, eyes, content->colour_conversion(), true, time);
288                 time += TIME_HZ / _film->video_frame_rate();
289         }
290
291         _last_emit_was_black = false;
292
293         _video_position = piece->video_position = time;
294 }
295
296 void
297 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
298 {
299         shared_ptr<Piece> piece = weak_piece.lock ();
300         if (!piece) {
301                 return;
302         }
303
304         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
305         assert (content);
306
307         /* Gain */
308         if (content->audio_gain() != 0) {
309                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
310                 gain->apply_gain (content->audio_gain ());
311                 audio = gain;
312         }
313
314         /* Resample */
315         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
316                 shared_ptr<Resampler> r = resampler (content, true);
317                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
318                 audio = ro.first;
319                 frame = ro.second;
320         }
321         
322         Time const relative_time = _film->audio_frames_to_time (frame);
323
324         if (content->trimmed (relative_time)) {
325                 return;
326         }
327
328         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time;
329         
330         /* Remap channels */
331         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
332         dcp_mapped->make_silent ();
333         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
334         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
335                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
336                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
337                 }
338         }
339
340         audio = dcp_mapped;
341
342         /* We must cut off anything that comes before the start of all time */
343         if (time < 0) {
344                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
345                 if (frames >= audio->frames ()) {
346                         return;
347                 }
348
349                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
350                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
351
352                 audio = trimmed;
353                 time = 0;
354         }
355
356         _audio_merger.push (audio, time);
357         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
358 }
359
360 void
361 Player::flush ()
362 {
363         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
364         if (tb.audio) {
365                 Audio (tb.audio, tb.time);
366                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
367         }
368
369         while (_video_position < _audio_position) {
370                 emit_black ();
371         }
372
373         while (_audio_position < _video_position) {
374                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
375         }
376         
377 }
378
379 /** Seek so that the next pass() will yield (approximately) the requested frame.
380  *  Pass accurate = true to try harder to get close to the request.
381  *  @return true on error
382  */
383 void
384 Player::seek (Time t, bool accurate)
385 {
386         if (!_have_valid_pieces) {
387                 setup_pieces ();
388                 _have_valid_pieces = true;
389         }
390
391         if (_pieces.empty ()) {
392                 return;
393         }
394
395         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
396                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
397                 if (!vc) {
398                         continue;
399                 }
400                 
401                 Time s = t - vc->position ();
402                 s = max (static_cast<Time> (0), s);
403                 s = min (vc->length_after_trim(), s);
404
405                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
406
407                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
408                 /* Here we are converting from time (in the DCP) to a frame number in the content.
409                    Hence we need to use the DCP's frame rate and the double/skip correction, not
410                    the source's rate.
411                 */
412                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
413                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
414         }
415
416         _video_position = _audio_position = t;
417         
418         /* XXX: don't seek audio because we don't need to... */
419 }
420
421 void
422 Player::setup_pieces ()
423 {
424         list<shared_ptr<Piece> > old_pieces = _pieces;
425
426         _pieces.clear ();
427
428         ContentList content = _playlist->content ();
429         sort (content.begin(), content.end(), ContentSorter ());
430
431         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
432
433                 shared_ptr<Piece> piece (new Piece (*i));
434
435                 /* XXX: into content? */
436
437                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
438                 if (fc) {
439                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
440                         
441                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
442                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
443                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
444
445                         piece->decoder = fd;
446                 }
447                 
448                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
449                 if (ic) {
450                         shared_ptr<StillImageDecoder> id;
451                         
452                         /* See if we can re-use an old StillImageDecoder */
453                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
454                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
455                                 if (imd && imd->content() == ic) {
456                                         id = imd;
457                                 }
458                         }
459
460                         if (!id) {
461                                 id.reset (new StillImageDecoder (_film, ic));
462                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
463                         }
464
465                         piece->decoder = id;
466                 }
467
468                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
469                 if (mc) {
470                         shared_ptr<MovingImageDecoder> md;
471
472                         if (!md) {
473                                 md.reset (new MovingImageDecoder (_film, mc));
474                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
475                         }
476
477                         piece->decoder = md;
478                 }
479
480                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
481                 if (sc) {
482                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
483                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
484
485                         piece->decoder = sd;
486                 }
487
488                 _pieces.push_back (piece);
489         }
490
491 #ifdef DEBUG_PLAYER
492         cout << "=== Player setup:\n";
493         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
494                 cout << *(i->get()) << "\n";
495         }
496 #endif  
497 }
498
499 void
500 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
501 {
502         shared_ptr<Content> c = w.lock ();
503         if (!c) {
504                 return;
505         }
506
507         if (
508                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
509                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
510                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO
511                 ) {
512                 
513                 _have_valid_pieces = false;
514                 Changed (frequent);
515
516         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
517                 update_subtitle ();
518                 Changed (frequent);
519         } else if (property == VideoContentProperty::VIDEO_FRAME_TYPE) {
520                 Changed (frequent);
521         }
522 }
523
524 void
525 Player::playlist_changed ()
526 {
527         _have_valid_pieces = false;
528         Changed (false);
529 }
530
531 void
532 Player::set_video_container_size (libdcp::Size s)
533 {
534         _video_container_size = s;
535         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
536         _black_frame->make_black ();
537 }
538
539 shared_ptr<Resampler>
540 Player::resampler (shared_ptr<AudioContent> c, bool create)
541 {
542         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
543         if (i != _resamplers.end ()) {
544                 return i->second;
545         }
546
547         if (!create) {
548                 return shared_ptr<Resampler> ();
549         }
550         
551         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
552         _resamplers[c] = r;
553         return r;
554 }
555
556 void
557 Player::emit_black ()
558 {
559 #ifdef DCPOMATIC_DEBUG
560         _last_video.reset ();
561 #endif
562         
563         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
564         _video_position += _film->video_frames_to_time (1);
565         _last_emit_was_black = true;
566 }
567
568 void
569 Player::emit_silence (OutputAudioFrame most)
570 {
571         if (most == 0) {
572                 return;
573         }
574         
575         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
576         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
577         silence->make_silent ();
578         Audio (silence, _audio_position);
579         _audio_position += _film->audio_frames_to_time (N);
580 }
581
582 void
583 Player::film_changed (Film::Property p)
584 {
585         /* Here we should notice Film properties that affect our output, and
586            alert listeners that our output now would be different to how it was
587            last time we were run.
588         */
589
590         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
591                 Changed (false);
592         }
593 }
594
595 void
596 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
597 {
598         _in_subtitle.piece = weak_piece;
599         _in_subtitle.image = image;
600         _in_subtitle.rect = rect;
601         _in_subtitle.from = from;
602         _in_subtitle.to = to;
603
604         update_subtitle ();
605 }
606
607 void
608 Player::update_subtitle ()
609 {
610         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
611         if (!piece) {
612                 return;
613         }
614
615         if (!_in_subtitle.image) {
616                 _out_subtitle.image.reset ();
617                 return;
618         }
619
620         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
621         assert (sc);
622
623         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
624         libdcp::Size scaled_size;
625
626         in_rect.y += sc->subtitle_offset ();
627
628         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
629         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
630         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
631
632         /* Then we need a corrective translation, consisting of two parts:
633          *
634          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
635          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
636          *
637          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
638          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
639          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
640          *
641          * Combining these two translations gives these expressions.
642          */
643         
644         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
645         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
646         
647         _out_subtitle.image = _in_subtitle.image->scale (
648                 scaled_size,
649                 Scaler::from_id ("bicubic"),
650                 _in_subtitle.image->pixel_format (),
651                 true
652                 );
653         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
654         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
655 }