2 Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
35 #include "raw_image_proxy.h"
38 #include "render_subtitles.h"
40 #include "content_video.h"
41 #include "player_video.h"
42 #include "frame_rate_change.h"
43 #include "dcp_content.h"
44 #include "dcp_decoder.h"
45 #include "dcp_subtitle_content.h"
46 #include "dcp_subtitle_decoder.h"
47 #include "audio_processor.h"
50 #include <dcp/reel_sound_asset.h>
51 #include <dcp/reel_subtitle_asset.h>
52 #include <dcp/reel_picture_asset.h>
53 #include <boost/foreach.hpp>
60 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
72 using boost::shared_ptr;
73 using boost::weak_ptr;
74 using boost::dynamic_pointer_cast;
75 using boost::optional;
77 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
79 , _playlist (playlist)
80 , _have_valid_pieces (false)
81 , _ignore_video (false)
82 , _ignore_audio (false)
83 , _always_burn_subtitles (false)
85 , _play_referenced (false)
87 _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
88 _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
89 _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
90 set_video_container_size (_film->frame_size ());
92 film_changed (Film::AUDIO_PROCESSOR);
96 Player::setup_pieces ()
98 list<shared_ptr<Piece> > old_pieces = _pieces;
101 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
103 if (!i->paths_valid ()) {
107 shared_ptr<Decoder> decoder;
108 optional<FrameRateChange> frc;
111 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (i);
113 decoder.reset (new FFmpegDecoder (fc, _film->log(), _fast));
114 frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
117 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (i);
119 decoder.reset (new DCPDecoder (dc, _fast));
120 frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
124 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (i);
126 /* See if we can re-use an old ImageDecoder */
127 for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
128 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
129 if (imd && imd->content() == ic) {
135 decoder.reset (new ImageDecoder (ic));
138 frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
142 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (i);
144 decoder.reset (new SndfileDecoder (sc, _fast));
146 /* Work out a FrameRateChange for the best overlap video for this content */
147 DCPTime best_overlap_t;
148 shared_ptr<VideoContent> best_overlap;
149 BOOST_FOREACH (shared_ptr<Content> j, _playlist->content ()) {
150 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (j);
155 DCPTime const overlap = min (vc->end(), i->end()) - max (vc->position(), i->position());
156 if (overlap > best_overlap_t) {
158 best_overlap_t = overlap;
163 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
165 /* No video overlap; e.g. if the DCP is just audio */
166 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
171 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (i);
173 decoder.reset (new SubRipDecoder (rc));
174 frc = FrameRateChange (rc->subtitle_video_frame_rate(), _film->video_frame_rate());
177 /* DCPSubtitleContent */
178 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (i);
180 decoder.reset (new DCPSubtitleDecoder (dsc));
181 frc = FrameRateChange (dsc->subtitle_video_frame_rate(), _film->video_frame_rate());
184 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
185 if (vd && _ignore_video) {
186 vd->set_ignore_video ();
189 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> (decoder);
190 if (ad && _ignore_audio) {
191 ad->set_ignore_audio ();
194 _pieces.push_back (shared_ptr<Piece> (new Piece (i, decoder, frc.get ())));
197 _have_valid_pieces = true;
201 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
203 shared_ptr<Content> c = w.lock ();
209 property == ContentProperty::POSITION ||
210 property == ContentProperty::LENGTH ||
211 property == ContentProperty::TRIM_START ||
212 property == ContentProperty::TRIM_END ||
213 property == ContentProperty::PATH ||
214 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
215 property == DCPContentProperty::CAN_BE_PLAYED
218 _have_valid_pieces = false;
222 property == SubtitleContentProperty::USE_SUBTITLES ||
223 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
224 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
225 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
226 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
227 property == SubtitleContentProperty::FONTS ||
228 property == VideoContentProperty::VIDEO_CROP ||
229 property == VideoContentProperty::VIDEO_SCALE ||
230 property == VideoContentProperty::VIDEO_FRAME_RATE ||
231 property == VideoContentProperty::VIDEO_FADE_IN ||
232 property == VideoContentProperty::VIDEO_FADE_OUT ||
233 property == VideoContentProperty::COLOUR_CONVERSION
241 Player::set_video_container_size (dcp::Size s)
243 _video_container_size = s;
245 _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
246 _black_image->make_black ();
250 Player::playlist_changed ()
252 _have_valid_pieces = false;
257 Player::film_changed (Film::Property p)
259 /* Here we should notice Film properties that affect our output, and
260 alert listeners that our output now would be different to how it was
261 last time we were run.
264 if (p == Film::CONTAINER) {
266 } else if (p == Film::VIDEO_FRAME_RATE) {
267 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
268 so we need new pieces here.
270 _have_valid_pieces = false;
272 } else if (p == Film::AUDIO_PROCESSOR) {
273 if (_film->audio_processor ()) {
274 _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
280 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
282 list<PositionImage> all;
284 for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
289 /* We will scale the subtitle up to fit _video_container_size */
290 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
292 /* Then we need a corrective translation, consisting of two parts:
294 * 1. that which is the result of the scaling of the subtitle by _video_container_size; this will be
295 * rect.x * _video_container_size.width and rect.y * _video_container_size.height.
297 * 2. that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
298 * (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
299 * (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
301 * Combining these two translations gives these expressions.
308 dcp::YUV_TO_RGB_REC601,
309 i->image->pixel_format (),
313 lrint (_video_container_size.width * i->rectangle.x),
314 lrint (_video_container_size.height * i->rectangle.y)
323 shared_ptr<PlayerVideo>
324 Player::black_player_video_frame (DCPTime time) const
326 return shared_ptr<PlayerVideo> (
328 shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
332 _video_container_size,
333 _video_container_size,
336 PresetColourConversion::all().front().conversion
341 /** @return All PlayerVideos at the given time. There may be none if the content
342 * at `time' is a DCP which we are passing through (i.e. referring to by reference)
343 * or 2 if we have 3D.
345 list<shared_ptr<PlayerVideo> >
346 Player::get_video (DCPTime time, bool accurate)
348 if (!_have_valid_pieces) {
352 /* Find subtitles for possible burn-in */
354 PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false, true);
356 list<PositionImage> sub_images;
358 /* Image subtitles */
359 list<PositionImage> c = transform_image_subtitles (ps.image);
360 copy (c.begin(), c.end(), back_inserter (sub_images));
362 /* Text subtitles (rendered to an image) */
363 if (!ps.text.empty ()) {
364 list<PositionImage> s = render_subtitles (ps.text, ps.fonts, _video_container_size);
365 copy (s.begin (), s.end (), back_inserter (sub_images));
368 optional<PositionImage> subtitles;
369 if (!sub_images.empty ()) {
370 subtitles = merge (sub_images);
373 /* Find pieces containing video which is happening now */
375 list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
377 time + DCPTime::from_frames (1, _film->video_frame_rate ()) - DCPTime::delta()
380 list<shared_ptr<PlayerVideo> > pvf;
383 /* No video content at this time */
384 pvf.push_back (black_player_video_frame (time));
386 /* Some video content at this time */
387 shared_ptr<Piece> last = *(ov.rbegin ());
388 VideoFrameType const last_type = dynamic_pointer_cast<VideoContent> (last->content)->video_frame_type ();
390 /* Get video from appropriate piece(s) */
391 BOOST_FOREACH (shared_ptr<Piece> piece, ov) {
393 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
394 DCPOMATIC_ASSERT (decoder);
395 shared_ptr<VideoContent> video_content = dynamic_pointer_cast<VideoContent> (piece->content);
396 DCPOMATIC_ASSERT (video_content);
398 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> (video_content);
399 if (dcp_content && dcp_content->reference_video () && !_play_referenced) {
404 /* always use the last video */
406 /* with a corresponding L/R eye if appropriate */
407 (last_type == VIDEO_FRAME_TYPE_3D_LEFT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) ||
408 (last_type == VIDEO_FRAME_TYPE_3D_RIGHT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_LEFT);
411 /* We want to use this piece */
412 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
413 if (content_video.empty ()) {
414 pvf.push_back (black_player_video_frame (time));
416 dcp::Size image_size = video_content->scale().size (video_content, _video_container_size, _film->frame_size ());
418 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
420 shared_ptr<PlayerVideo> (
423 content_video_to_dcp (piece, i->frame),
424 video_content->crop (),
425 video_content->fade (i->frame),
427 _video_container_size,
430 video_content->colour_conversion ()
437 /* Discard unused video */
438 decoder->get_video (dcp_to_content_video (piece, time), accurate);
444 BOOST_FOREACH (shared_ptr<PlayerVideo> p, pvf) {
445 p->set_subtitle (subtitles.get ());
452 /** @return Audio data or 0 if the only audio data here is referenced DCP data */
453 shared_ptr<AudioBuffers>
454 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
456 if (!_have_valid_pieces) {
460 Frame const length_frames = length.frames_round (_film->audio_frame_rate ());
462 shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
463 audio->make_silent ();
465 list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
470 bool all_referenced = true;
471 BOOST_FOREACH (shared_ptr<Piece> i, ov) {
472 shared_ptr<AudioContent> audio_content = dynamic_pointer_cast<AudioContent> (i->content);
473 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> (i->content);
474 if (audio_content && (!dcp_content || !dcp_content->reference_audio ())) {
475 /* There is audio content which is not from a DCP or not set to be referenced */
476 all_referenced = false;
480 if (all_referenced && !_play_referenced) {
481 return shared_ptr<AudioBuffers> ();
484 BOOST_FOREACH (shared_ptr<Piece> i, ov) {
486 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (i->content);
487 DCPOMATIC_ASSERT (content);
488 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> (i->decoder);
489 DCPOMATIC_ASSERT (decoder);
491 /* The time that we should request from the content */
492 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
493 Frame request_frames = length_frames;
495 if (request < DCPTime ()) {
496 /* We went off the start of the content, so we will need to offset
497 the stuff we get back.
500 request_frames += request.frames_round (_film->audio_frame_rate ());
501 if (request_frames < 0) {
504 request = DCPTime ();
507 Frame const content_frame = dcp_to_resampled_audio (i, request);
509 BOOST_FOREACH (AudioStreamPtr j, content->audio_streams ()) {
511 if (j->channels() == 0) {
512 /* Some content (e.g. DCPs) can have streams with no channels */
516 /* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
517 ContentAudio all = decoder->get_audio (j, content_frame, request_frames, accurate);
520 if (content->audio_gain() != 0) {
521 shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
522 gain->apply_gain (content->audio_gain ());
527 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
528 dcp_mapped->make_silent ();
529 AudioMapping map = j->mapping ();
530 for (int i = 0; i < map.input_channels(); ++i) {
531 for (int j = 0; j < _film->audio_channels(); ++j) {
532 if (map.get (i, j) > 0) {
533 dcp_mapped->accumulate_channel (
543 if (_audio_processor) {
544 dcp_mapped = _audio_processor->run (dcp_mapped, _film->audio_channels ());
547 all.audio = dcp_mapped;
549 audio->accumulate_frames (
551 content_frame - all.frame,
552 offset.frames_round (_film->audio_frame_rate()),
553 min (Frame (all.audio->frames()), request_frames)
562 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
564 shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
565 DCPTime s = t - piece->content->position ();
566 s = min (piece->content->length_after_trim(), s);
567 s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
569 /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
570 then convert that ContentTime to frames at the content's rate. However this fails for
571 situations like content at 29.9978733fps, DCP at 30fps. The accuracy of the Time type is not
572 enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
574 Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
576 return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
580 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
582 shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
583 /* See comment in dcp_to_content_video */
584 DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime (piece->content->trim_start (), piece->frc);
585 return max (DCPTime (), d + piece->content->position ());
589 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
591 DCPTime s = t - piece->content->position ();
592 s = min (piece->content->length_after_trim(), s);
593 /* See notes in dcp_to_content_video */
594 return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
598 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
600 DCPTime s = t - piece->content->position ();
601 s = min (piece->content->length_after_trim(), s);
602 return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
606 Player::content_subtitle_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
608 return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
611 /** @param burnt true to return only subtitles to be burnt, false to return only
612 * subtitles that should not be burnt. This parameter will be ignored if
613 * _always_burn_subtitles is true; in this case, all subtitles will be returned.
616 Player::get_subtitles (DCPTime time, DCPTime length, bool starting, bool burnt)
618 list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
620 PlayerSubtitles ps (time, length);
622 for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
623 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
624 if (!subtitle_content->use_subtitles () || (!_always_burn_subtitles && (burnt != subtitle_content->burn_subtitles ()))) {
628 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> (subtitle_content);
629 if (dcp_content && dcp_content->reference_subtitle () && !_play_referenced) {
633 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
634 ContentTime const from = dcp_to_content_subtitle (*j, time);
635 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
636 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
638 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
639 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
641 /* Apply content's subtitle offsets */
642 i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
643 i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
645 /* Apply content's subtitle scale */
646 i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
647 i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
649 /* Apply a corrective translation to keep the subtitle centred after that scale */
650 i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
651 i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
653 ps.image.push_back (i->sub);
656 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
657 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
658 BOOST_FOREACH (dcp::SubtitleString s, ts.subs) {
659 s.set_h_position (s.h_position() + subtitle_content->subtitle_x_offset ());
660 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
661 float const xs = subtitle_content->subtitle_x_scale();
662 float const ys = subtitle_content->subtitle_y_scale();
663 float const average = s.size() * (xs + ys) / 2;
664 s.set_size (average);
665 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
666 s.set_aspect_adjust (xs / ys);
668 s.set_in (dcp::Time(content_subtitle_to_dcp (*j, ts.period().from).seconds()));
669 s.set_out (dcp::Time(content_subtitle_to_dcp (*j, ts.period().to).seconds()));
670 ps.text.push_back (s);
671 ps.add_fonts (subtitle_content->fonts ());
679 list<shared_ptr<Font> >
680 Player::get_subtitle_fonts ()
682 if (!_have_valid_pieces) {
686 list<shared_ptr<Font> > fonts;
687 BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
688 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
690 /* XXX: things may go wrong if there are duplicate font IDs
691 with different font files.
693 list<shared_ptr<Font> > f = sc->fonts ();
694 copy (f.begin(), f.end(), back_inserter (fonts));
701 /** Set this player never to produce any video data */
703 Player::set_ignore_video ()
705 _ignore_video = true;
708 /** Set this player never to produce any audio data */
710 Player::set_ignore_audio ()
712 _ignore_audio = true;
715 /** Set whether or not this player should always burn text subtitles into the image,
716 * regardless of the content settings.
717 * @param burn true to always burn subtitles, false to obey content settings.
720 Player::set_always_burn_subtitles (bool burn)
722 _always_burn_subtitles = burn;
729 _have_valid_pieces = false;
733 Player::set_play_referenced ()
735 _play_referenced = true;
736 _have_valid_pieces = false;
739 list<shared_ptr<dcp::ReelAsset> >
740 Player::get_reel_assets ()
742 list<shared_ptr<dcp::ReelAsset> > a;
744 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
745 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
749 /* XXX: hack hack hack */
750 DCPDecoder decoder (j, false);
751 if (j->reference_video ()) {
752 a.push_back (decoder.reels().front()->main_picture ());
754 if (j->reference_audio ()) {
755 a.push_back (decoder.reels().front()->main_sound ());
757 if (j->reference_subtitle ()) {
758 a.push_back (decoder.reels().front()->main_subtitle ());