2 Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24 #include "ffmpeg_decoder.h"
25 #include "audio_buffers.h"
26 #include "ffmpeg_content.h"
27 #include "image_decoder.h"
28 #include "image_content.h"
29 #include "sndfile_decoder.h"
30 #include "sndfile_content.h"
31 #include "subtitle_content.h"
32 #include "subrip_decoder.h"
33 #include "subrip_content.h"
40 #include "render_subtitles.h"
49 using boost::shared_ptr;
50 using boost::weak_ptr;
51 using boost::dynamic_pointer_cast;
52 using boost::optional;
57 Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
63 shared_ptr<Content> content;
64 shared_ptr<Decoder> decoder;
68 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
73 , _have_valid_pieces (false)
76 , _audio_merger (f->audio_channels(), f->audio_frame_rate ())
77 , _last_emit_was_black (false)
78 , _just_did_inaccurate_seek (false)
79 , _approximate_size (false)
81 _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
82 _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
83 _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
84 set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
88 Player::disable_video ()
94 Player::disable_audio ()
102 if (!_have_valid_pieces) {
106 /* Interrogate all our pieces to find the one with the earliest decoded data */
108 shared_ptr<Piece> earliest_piece;
109 shared_ptr<Decoded> earliest_decoded;
110 DCPTime earliest_time = DCPTime::max ();
111 DCPTime earliest_audio = DCPTime::max ();
113 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
115 DCPTime const offset = (*i)->content->position() - (*i)->content->trim_start();
118 shared_ptr<Decoded> dec;
120 dec = (*i)->decoder->peek ();
122 /* Decoder has nothing else to give us */
127 dec->set_dcp_times ((*i)->frc, offset);
128 DCPTime const t = dec->dcp_time - offset;
129 cout << "Peeked " << (*i)->content->paths()[0] << " for " << t << " cf " << ((*i)->content->full_length() - (*i)->content->trim_end ()) << "\n";
130 if (t >= ((*i)->content->full_length() - (*i)->content->trim_end ())) {
131 /* In the end-trimmed part; decoder has nothing else to give us */
134 } else if (t >= (*i)->content->trim_start ()) {
135 /* Within the un-trimmed part; everything's ok */
138 /* Within the start-trimmed part; get something else */
139 (*i)->decoder->consume ();
147 if (dec->dcp_time < earliest_time) {
149 earliest_decoded = dec;
150 earliest_time = dec->dcp_time;
153 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
154 earliest_audio = dec->dcp_time;
158 if (!earliest_piece) {
163 if (earliest_audio != DCPTime::max ()) {
164 if (earliest_audio.get() < 0) {
165 earliest_audio = DCPTime ();
167 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (earliest_audio);
168 Audio (tb.audio, tb.time);
169 /* This assumes that the audio-frames-to-time conversion is exact
170 so that there are no accumulated errors caused by rounding.
172 _audio_position += DCPTime::from_frames (tb.audio->frames(), _film->audio_frame_rate ());
175 /* Emit the earliest thing */
177 shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
178 shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
179 shared_ptr<DecodedImageSubtitle> dis = dynamic_pointer_cast<DecodedImageSubtitle> (earliest_decoded);
180 shared_ptr<DecodedTextSubtitle> dts = dynamic_pointer_cast<DecodedTextSubtitle> (earliest_decoded);
182 /* Will be set to false if we shouldn't consume the peeked DecodedThing */
187 if (_just_did_inaccurate_seek) {
189 /* Just emit; no subtlety */
190 emit_video (earliest_piece, dv);
191 step_video_position (dv);
193 } else if (dv->dcp_time > _video_position) {
197 list<shared_ptr<Piece> >::iterator i = _pieces.begin();
198 while (i != _pieces.end() && ((*i)->content->position() >= _video_position || _video_position >= (*i)->content->end())) {
202 if (i == _pieces.end() || !_last_incoming_video.video || !_have_valid_pieces) {
203 /* We're outside all video content */
205 _statistics.video.black++;
207 /* We're inside some video; repeat the frame */
208 _last_incoming_video.video->dcp_time = _video_position;
209 emit_video (_last_incoming_video.weak_piece, _last_incoming_video.video);
210 step_video_position (_last_incoming_video.video);
211 _statistics.video.repeat++;
216 } else if (dv->dcp_time == _video_position) {
218 emit_video (earliest_piece, dv);
219 step_video_position (dv);
220 _statistics.video.good++;
222 /* Too far behind: skip */
223 _statistics.video.skip++;
226 _just_did_inaccurate_seek = false;
228 } else if (da && _audio) {
230 if (da->dcp_time > _audio_position) {
232 emit_silence (da->dcp_time - _audio_position);
234 _statistics.audio.silence += (da->dcp_time - _audio_position);
235 } else if (da->dcp_time == _audio_position) {
237 emit_audio (earliest_piece, da);
238 _statistics.audio.good += da->data->frames();
240 /* Too far behind: skip */
241 _statistics.audio.skip += da->data->frames();
244 } else if (dis && _video) {
245 _image_subtitle.piece = earliest_piece;
246 _image_subtitle.subtitle = dis;
247 update_subtitle_from_image ();
248 } else if (dts && _video) {
249 _text_subtitle.piece = earliest_piece;
250 _text_subtitle.subtitle = dts;
251 update_subtitle_from_text ();
255 earliest_piece->decoder->consume ();
262 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
264 /* Keep a note of what came in so that we can repeat it if required */
265 _last_incoming_video.weak_piece = weak_piece;
266 _last_incoming_video.video = video;
268 shared_ptr<Piece> piece = weak_piece.lock ();
273 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
276 FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
278 dcp::Size image_size = content->scale().size (content, _video_container_size);
279 if (_approximate_size) {
280 image_size.width &= ~3;
281 image_size.height &= ~3;
284 shared_ptr<PlayerImage> pi (
289 _video_container_size,
295 _film->with_subtitles () &&
296 _out_subtitle.image &&
297 video->dcp_time >= _out_subtitle.from && video->dcp_time <= _out_subtitle.to
300 Position<int> const container_offset (
301 (_video_container_size.width - image_size.width) / 2,
302 (_video_container_size.height - image_size.height) / 2
305 pi->set_subtitle (_out_subtitle.image, _out_subtitle.position + container_offset);
309 #ifdef DCPOMATIC_DEBUG
310 _last_video = piece->content;
313 Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
315 _last_emit_was_black = false;
319 Player::step_video_position (shared_ptr<DecodedVideo> video)
321 /* This is a bit of a hack; don't update _video_position if EYES_RIGHT is on its way */
322 if (video->eyes != EYES_LEFT) {
323 /* This assumes that the video-frames-to-time conversion is exact
324 so that there are no accumulated errors caused by rounding.
326 _video_position += DCPTime::from_frames (1, _film->video_frame_rate ());
331 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
333 shared_ptr<Piece> piece = weak_piece.lock ();
338 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
342 if (content->audio_gain() != 0) {
343 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
344 gain->apply_gain (content->audio_gain ());
349 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
350 dcp_mapped->make_silent ();
351 AudioMapping map = content->audio_mapping ();
352 for (int i = 0; i < map.content_channels(); ++i) {
353 for (int j = 0; j < _film->audio_channels(); ++j) {
354 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
355 dcp_mapped->accumulate_channel (
358 static_cast<dcp::Channel> (j),
359 map.get (i, static_cast<dcp::Channel> (j))
365 audio->data = dcp_mapped;
368 audio->dcp_time += DCPTime::from_seconds (content->audio_delay() / 1000.0);
369 if (audio->dcp_time < DCPTime (0)) {
370 int const frames = - audio->dcp_time.frames (_film->audio_frame_rate());
371 if (frames >= audio->data->frames ()) {
375 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
376 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
378 audio->data = trimmed;
379 audio->dcp_time = DCPTime ();
382 _audio_merger.push (audio->data, audio->dcp_time);
388 TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
389 if (_audio && tb.audio) {
390 Audio (tb.audio, tb.time);
391 _audio_position += DCPTime::from_frames (tb.audio->frames (), _film->audio_frame_rate ());
394 while (_video && _video_position < _audio_position) {
398 while (_audio && _audio_position < _video_position) {
399 emit_silence (_video_position - _audio_position);
403 /** Seek so that the next pass() will yield (approximately) the requested frame.
404 * Pass accurate = true to try harder to get close to the request.
405 * @return true on error
408 Player::seek (DCPTime t, bool accurate)
410 if (!_have_valid_pieces) {
414 if (_pieces.empty ()) {
418 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
419 /* s is the offset of t from the start position of this content */
420 DCPTime s = t - (*i)->content->position ();
421 s = max (static_cast<DCPTime> (0), s);
422 s = min ((*i)->content->length_after_trim(), s);
424 /* Convert this to the content time */
425 ContentTime ct (s + (*i)->content->trim_start(), (*i)->frc);
427 /* And seek the decoder */
428 (*i)->decoder->seek (ct, accurate);
431 _video_position = t.round_up (_film->video_frame_rate());
432 _audio_position = t.round_up (_film->audio_frame_rate());
434 _audio_merger.clear (_audio_position);
437 /* We just did an inaccurate seek, so it's likely that the next thing seen
438 out of pass() will be a fair distance from _{video,audio}_position. Setting
439 this flag stops pass() from trying to fix that: we assume that if it
440 was an inaccurate seek then the caller does not care too much about
441 inserting black/silence to keep the time tidy.
443 _just_did_inaccurate_seek = true;
448 Player::setup_pieces ()
450 list<shared_ptr<Piece> > old_pieces = _pieces;
453 ContentList content = _playlist->content ();
455 for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
457 if (!(*i)->paths_valid ()) {
461 shared_ptr<Decoder> decoder;
462 optional<FrameRateChange> frc;
464 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
465 DCPTime best_overlap_t;
466 shared_ptr<VideoContent> best_overlap;
467 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
468 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
473 DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
474 if (overlap > best_overlap_t) {
476 best_overlap_t = overlap;
480 optional<FrameRateChange> best_overlap_frc;
482 best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
484 /* No video overlap; e.g. if the DCP is just audio */
485 best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
489 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
491 decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
492 frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
496 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
498 /* See if we can re-use an old ImageDecoder */
499 for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
500 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
501 if (imd && imd->content() == ic) {
507 decoder.reset (new ImageDecoder (_film, ic));
510 frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
514 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
516 decoder.reset (new SndfileDecoder (_film, sc));
517 frc = best_overlap_frc;
521 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
523 decoder.reset (new SubRipDecoder (_film, rc));
524 frc = best_overlap_frc;
527 ContentTime st ((*i)->trim_start(), frc.get ());
528 decoder->seek (st, true);
530 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
533 _have_valid_pieces = true;
535 /* The Piece for the _last_incoming_video will no longer be valid */
536 _last_incoming_video.video.reset ();
538 _video_position = DCPTime ();
539 _audio_position = DCPTime ();
543 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
545 shared_ptr<Content> c = w.lock ();
551 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
552 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
553 property == VideoContentProperty::VIDEO_FRAME_TYPE
556 _have_valid_pieces = false;
560 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
561 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
562 property == SubtitleContentProperty::SUBTITLE_SCALE
565 update_subtitle_from_image ();
566 update_subtitle_from_text ();
570 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_SCALE ||
571 property == VideoContentProperty::VIDEO_FRAME_RATE
576 } else if (property == ContentProperty::PATH) {
578 _have_valid_pieces = false;
584 Player::playlist_changed ()
586 _have_valid_pieces = false;
591 Player::set_video_container_size (dcp::Size s)
593 _video_container_size = s;
595 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
602 _video_container_size,
603 _video_container_size,
604 Scaler::from_id ("bicubic")
610 Player::emit_black ()
612 #ifdef DCPOMATIC_DEBUG
613 _last_video.reset ();
616 Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
617 _video_position += DCPTime::from_frames (1, _film->video_frame_rate ());
618 _last_emit_was_black = true;
622 Player::emit_silence (DCPTime most)
628 DCPTime t = min (most, DCPTime::from_seconds (0.5));
629 shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t.frames (_film->audio_frame_rate())));
630 silence->make_silent ();
631 Audio (silence, _audio_position);
633 _audio_position += t;
637 Player::film_changed (Film::Property p)
639 /* Here we should notice Film properties that affect our output, and
640 alert listeners that our output now would be different to how it was
641 last time we were run.
644 if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
650 Player::update_subtitle_from_image ()
652 shared_ptr<Piece> piece = _image_subtitle.piece.lock ();
657 if (!_image_subtitle.subtitle->image) {
658 _out_subtitle.image.reset ();
662 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
665 dcpomatic::Rect<double> in_rect = _image_subtitle.subtitle->rect;
666 dcp::Size scaled_size;
668 in_rect.x += sc->subtitle_x_offset ();
669 in_rect.y += sc->subtitle_y_offset ();
671 /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
672 scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
673 scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
675 /* Then we need a corrective translation, consisting of two parts:
677 * 1. that which is the result of the scaling of the subtitle by _video_container_size; this will be
678 * rect.x * _video_container_size.width and rect.y * _video_container_size.height.
680 * 2. that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
681 * (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
682 * (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
684 * Combining these two translations gives these expressions.
687 _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
688 _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
690 _out_subtitle.image = _image_subtitle.subtitle->image->scale (
692 Scaler::from_id ("bicubic"),
693 _image_subtitle.subtitle->image->pixel_format (),
697 _out_subtitle.from = _image_subtitle.subtitle->dcp_time + piece->content->position ();
698 _out_subtitle.to = _image_subtitle.subtitle->dcp_time_to + piece->content->position ();
701 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
702 * @return false if this could not be done.
705 Player::repeat_last_video ()
707 if (!_last_incoming_video.video || !_have_valid_pieces) {
712 _last_incoming_video.weak_piece,
713 _last_incoming_video.video
720 Player::update_subtitle_from_text ()
722 if (_text_subtitle.subtitle->subs.empty ()) {
723 _out_subtitle.image.reset ();
727 render_subtitles (_text_subtitle.subtitle->subs, _video_container_size, _out_subtitle.image, _out_subtitle.position);
731 Player::set_approximate_size ()
733 _approximate_size = true;
736 PlayerImage::PlayerImage (
737 shared_ptr<const Image> in,
739 dcp::Size inter_size,
741 Scaler const * scaler
745 , _inter_size (inter_size)
746 , _out_size (out_size)
753 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
755 _subtitle_image = image;
756 _subtitle_position = pos;
760 PlayerImage::image (AVPixelFormat format, bool aligned)
762 shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, format, aligned);
764 Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
766 if (_subtitle_image) {
767 out->alpha_blend (_subtitle_image, _subtitle_position);
774 PlayerStatistics::dump (shared_ptr<Log> log) const
776 log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat));
777 log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence));
780 PlayerStatistics const &
781 Player::statistics () const