2 Copyright (C) 2013-2016 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
23 #include "audio_buffers.h"
24 #include "content_audio.h"
25 #include "dcp_content.h"
28 #include "raw_image_proxy.h"
31 #include "render_subtitles.h"
33 #include "content_video.h"
34 #include "player_video.h"
35 #include "frame_rate_change.h"
36 #include "audio_processor.h"
38 #include "referenced_reel_asset.h"
39 #include "decoder_factory.h"
41 #include "video_decoder.h"
42 #include "audio_decoder.h"
43 #include "subtitle_content.h"
44 #include "subtitle_decoder.h"
45 #include "ffmpeg_content.h"
46 #include "audio_content.h"
47 #include "content_subtitle.h"
48 #include "dcp_decoder.h"
49 #include "image_decoder.h"
51 #include <dcp/reel_sound_asset.h>
52 #include <dcp/reel_subtitle_asset.h>
53 #include <dcp/reel_picture_asset.h>
54 #include <boost/foreach.hpp>
61 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
73 using boost::shared_ptr;
74 using boost::weak_ptr;
75 using boost::dynamic_pointer_cast;
76 using boost::optional;
77 using boost::scoped_ptr;
80 has_video (Content* c)
82 return static_cast<bool>(c->video);
86 has_audio (Content* c)
88 return static_cast<bool>(c->audio);
92 has_subtitle (Content* c)
94 return static_cast<bool>(c->subtitle);
97 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
99 , _playlist (playlist)
100 , _have_valid_pieces (false)
101 , _ignore_video (false)
102 , _ignore_audio (false)
103 , _always_burn_subtitles (false)
105 , _play_referenced (false)
107 _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
108 _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
109 _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
110 set_video_container_size (_film->frame_size ());
112 film_changed (Film::AUDIO_PROCESSOR);
116 Player::setup_pieces ()
120 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
122 if (!i->paths_valid ()) {
126 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log());
127 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
130 /* Not something that we can decode; e.g. Atmos content */
134 if (decoder->video && _ignore_video) {
135 decoder->video->set_ignore ();
138 if (decoder->audio && _ignore_audio) {
139 decoder->audio->set_ignore ();
142 if (decoder->audio && _fast) {
143 decoder->audio->set_fast ();
146 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
147 if (dcp && _play_referenced) {
148 dcp->set_decode_referenced ();
151 _pieces.push_back (shared_ptr<Piece> (new Piece (i, decoder, frc)));
154 _have_valid_pieces = true;
158 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
160 shared_ptr<Content> c = w.lock ();
166 property == ContentProperty::POSITION ||
167 property == ContentProperty::LENGTH ||
168 property == ContentProperty::TRIM_START ||
169 property == ContentProperty::TRIM_END ||
170 property == ContentProperty::PATH ||
171 property == VideoContentProperty::FRAME_TYPE ||
172 property == DCPContentProperty::CAN_BE_PLAYED ||
173 property == SubtitleContentProperty::COLOUR ||
174 property == SubtitleContentProperty::OUTLINE ||
175 property == SubtitleContentProperty::SHADOW ||
176 property == SubtitleContentProperty::EFFECT_COLOUR ||
177 property == FFmpegContentProperty::SUBTITLE_STREAM ||
178 property == VideoContentProperty::COLOUR_CONVERSION
181 _have_valid_pieces = false;
185 property == SubtitleContentProperty::LINE_SPACING ||
186 property == SubtitleContentProperty::OUTLINE_WIDTH ||
187 property == SubtitleContentProperty::Y_SCALE
190 /* These changes just need the pieces' decoders to be reset.
191 It's quite possible that other changes could be handled by
192 this branch rather than the _have_valid_pieces = false branch
193 above. This would make things a lot faster.
200 property == ContentProperty::VIDEO_FRAME_RATE ||
201 property == SubtitleContentProperty::USE ||
202 property == SubtitleContentProperty::X_OFFSET ||
203 property == SubtitleContentProperty::Y_OFFSET ||
204 property == SubtitleContentProperty::X_SCALE ||
205 property == SubtitleContentProperty::FONTS ||
206 property == VideoContentProperty::CROP ||
207 property == VideoContentProperty::SCALE ||
208 property == VideoContentProperty::FADE_IN ||
209 property == VideoContentProperty::FADE_OUT
217 Player::set_video_container_size (dcp::Size s)
219 _video_container_size = s;
221 _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
222 _black_image->make_black ();
226 Player::playlist_changed ()
228 _have_valid_pieces = false;
233 Player::film_changed (Film::Property p)
235 /* Here we should notice Film properties that affect our output, and
236 alert listeners that our output now would be different to how it was
237 last time we were run.
240 if (p == Film::CONTAINER) {
242 } else if (p == Film::VIDEO_FRAME_RATE) {
243 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
244 so we need new pieces here.
246 _have_valid_pieces = false;
248 } else if (p == Film::AUDIO_PROCESSOR) {
249 if (_film->audio_processor ()) {
250 _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
256 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
258 list<PositionImage> all;
260 for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
265 /* We will scale the subtitle up to fit _video_container_size */
266 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
268 /* Then we need a corrective translation, consisting of two parts:
270 * 1. that which is the result of the scaling of the subtitle by _video_container_size; this will be
271 * rect.x * _video_container_size.width and rect.y * _video_container_size.height.
273 * 2. that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
274 * (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
275 * (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
277 * Combining these two translations gives these expressions.
284 dcp::YUV_TO_RGB_REC601,
285 i->image->pixel_format (),
290 lrint (_video_container_size.width * i->rectangle.x),
291 lrint (_video_container_size.height * i->rectangle.y)
300 shared_ptr<PlayerVideo>
301 Player::black_player_video_frame (DCPTime time) const
303 return shared_ptr<PlayerVideo> (
305 shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
309 _video_container_size,
310 _video_container_size,
313 PresetColourConversion::all().front().conversion
318 /** @return All PlayerVideos at the given time. There may be none if the content
319 * at `time' is a DCP which we are passing through (i.e. referring to by reference)
320 * or 2 if we have 3D.
322 list<shared_ptr<PlayerVideo> >
323 Player::get_video (DCPTime time, bool accurate)
325 if (!_have_valid_pieces) {
329 /* Find subtitles for possible burn-in */
331 PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false, true, accurate);
333 list<PositionImage> sub_images;
335 /* Image subtitles */
336 list<PositionImage> c = transform_image_subtitles (ps.image);
337 copy (c.begin(), c.end(), back_inserter (sub_images));
339 /* Text subtitles (rendered to an image) */
340 if (!ps.text.empty ()) {
341 list<PositionImage> s = render_subtitles (ps.text, ps.fonts, _video_container_size, time);
342 copy (s.begin (), s.end (), back_inserter (sub_images));
345 optional<PositionImage> subtitles;
346 if (!sub_images.empty ()) {
347 subtitles = merge (sub_images);
350 /* Find pieces containing video which is happening now */
352 list<shared_ptr<Piece> > ov = overlaps (
354 time + DCPTime::from_frames (1, _film->video_frame_rate ()),
358 list<shared_ptr<PlayerVideo> > pvf;
361 /* No video content at this time */
362 pvf.push_back (black_player_video_frame (time));
364 /* Some video content at this time */
365 shared_ptr<Piece> last = *(ov.rbegin ());
366 VideoFrameType const last_type = last->content->video->frame_type ();
368 /* Get video from appropriate piece(s) */
369 BOOST_FOREACH (shared_ptr<Piece> piece, ov) {
371 shared_ptr<VideoDecoder> decoder = piece->decoder->video;
372 DCPOMATIC_ASSERT (decoder);
374 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> (piece->content);
375 if (dcp_content && dcp_content->reference_video () && !_play_referenced) {
380 /* always use the last video */
382 /* with a corresponding L/R eye if appropriate */
383 (last_type == VIDEO_FRAME_TYPE_3D_LEFT && piece->content->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) ||
384 (last_type == VIDEO_FRAME_TYPE_3D_RIGHT && piece->content->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT);
387 /* We want to use this piece */
388 list<ContentVideo> content_video = decoder->get (dcp_to_content_video (piece, time), accurate);
389 if (content_video.empty ()) {
390 pvf.push_back (black_player_video_frame (time));
392 dcp::Size image_size = piece->content->video->scale().size (
393 piece->content->video, _video_container_size, _film->frame_size ()
396 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
398 shared_ptr<PlayerVideo> (
402 piece->content->video->crop (),
403 piece->content->video->fade (i->frame.index()),
405 _video_container_size,
408 piece->content->video->colour_conversion ()
415 /* Discard unused video */
416 decoder->get (dcp_to_content_video (piece, time), accurate);
422 BOOST_FOREACH (shared_ptr<PlayerVideo> p, pvf) {
423 p->set_subtitle (subtitles.get ());
430 /** @return Audio data or 0 if the only audio data here is referenced DCP data */
431 shared_ptr<AudioBuffers>
432 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
434 if (!_have_valid_pieces) {
438 Frame const length_frames = length.frames_round (_film->audio_frame_rate ());
440 shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
441 audio->make_silent ();
443 list<shared_ptr<Piece> > ov = overlaps (time, time + length, has_audio);
448 bool all_referenced = true;
449 BOOST_FOREACH (shared_ptr<Piece> i, ov) {
450 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> (i->content);
451 if (i->content->audio && (!dcp_content || !dcp_content->reference_audio ())) {
452 /* There is audio content which is not from a DCP or not set to be referenced */
453 all_referenced = false;
457 if (all_referenced && !_play_referenced) {
458 return shared_ptr<AudioBuffers> ();
461 BOOST_FOREACH (shared_ptr<Piece> i, ov) {
463 DCPOMATIC_ASSERT (i->content->audio);
464 shared_ptr<AudioDecoder> decoder = i->decoder->audio;
465 DCPOMATIC_ASSERT (decoder);
467 /* The time that we should request from the content */
468 DCPTime request = time - DCPTime::from_seconds (i->content->audio->delay() / 1000.0);
469 Frame request_frames = length_frames;
471 if (request < DCPTime ()) {
472 /* We went off the start of the content, so we will need to offset
473 the stuff we get back.
476 request_frames += request.frames_round (_film->audio_frame_rate ());
477 if (request_frames < 0) {
480 request = DCPTime ();
483 Frame const content_frame = dcp_to_resampled_audio (i, request);
485 BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams ()) {
487 if (j->channels() == 0) {
488 /* Some content (e.g. DCPs) can have streams with no channels */
492 /* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
493 ContentAudio all = decoder->get (j, content_frame, request_frames, accurate);
496 if (i->content->audio->gain() != 0) {
497 shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
498 gain->apply_gain (i->content->audio->gain ());
503 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
504 dcp_mapped->make_silent ();
505 AudioMapping map = j->mapping ();
506 for (int i = 0; i < map.input_channels(); ++i) {
507 for (int j = 0; j < _film->audio_channels(); ++j) {
508 if (map.get (i, j) > 0) {
509 dcp_mapped->accumulate_channel (
519 if (_audio_processor) {
520 dcp_mapped = _audio_processor->run (dcp_mapped, _film->audio_channels ());
523 all.audio = dcp_mapped;
525 audio->accumulate_frames (
527 content_frame - all.frame,
528 offset.frames_round (_film->audio_frame_rate()),
529 min (Frame (all.audio->frames()), request_frames)
538 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
540 DCPTime s = t - piece->content->position ();
541 s = min (piece->content->length_after_trim(), s);
542 s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
544 /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
545 then convert that ContentTime to frames at the content's rate. However this fails for
546 situations like content at 29.9978733fps, DCP at 30fps. The accuracy of the Time type is not
547 enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
549 Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
551 return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
555 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
557 /* See comment in dcp_to_content_video */
558 DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime (piece->content->trim_start (), piece->frc);
559 return max (DCPTime (), d + piece->content->position ());
563 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
565 DCPTime s = t - piece->content->position ();
566 s = min (piece->content->length_after_trim(), s);
567 /* See notes in dcp_to_content_video */
568 return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
572 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
574 DCPTime s = t - piece->content->position ();
575 s = min (piece->content->length_after_trim(), s);
576 return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
580 Player::content_subtitle_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
582 return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
585 /** @param burnt true to return only subtitles to be burnt, false to return only
586 * subtitles that should not be burnt. This parameter will be ignored if
587 * _always_burn_subtitles is true; in this case, all subtitles will be returned.
590 Player::get_subtitles (DCPTime time, DCPTime length, bool starting, bool burnt, bool accurate)
592 list<shared_ptr<Piece> > subs = overlaps (time, time + length, has_subtitle);
594 PlayerSubtitles ps (time);
596 for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
597 if (!(*j)->content->subtitle->use () || (!_always_burn_subtitles && (burnt != (*j)->content->subtitle->burn ()))) {
601 shared_ptr<DCPContent> dcp_content = dynamic_pointer_cast<DCPContent> ((*j)->content);
602 if (dcp_content && dcp_content->reference_subtitle () && !_play_referenced) {
606 shared_ptr<SubtitleDecoder> subtitle_decoder = (*j)->decoder->subtitle;
607 ContentTime const from = dcp_to_content_subtitle (*j, time);
608 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
609 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
611 list<ContentImageSubtitle> image = subtitle_decoder->get_image (ContentTimePeriod (from, to), starting, accurate);
612 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
614 /* Apply content's subtitle offsets */
615 i->sub.rectangle.x += (*j)->content->subtitle->x_offset ();
616 i->sub.rectangle.y += (*j)->content->subtitle->y_offset ();
618 /* Apply content's subtitle scale */
619 i->sub.rectangle.width *= (*j)->content->subtitle->x_scale ();
620 i->sub.rectangle.height *= (*j)->content->subtitle->y_scale ();
622 /* Apply a corrective translation to keep the subtitle centred after that scale */
623 i->sub.rectangle.x -= i->sub.rectangle.width * ((*j)->content->subtitle->x_scale() - 1);
624 i->sub.rectangle.y -= i->sub.rectangle.height * ((*j)->content->subtitle->y_scale() - 1);
626 ps.image.push_back (i->sub);
629 list<ContentTextSubtitle> text = subtitle_decoder->get_text (ContentTimePeriod (from, to), starting, accurate);
630 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
631 BOOST_FOREACH (dcp::SubtitleString s, ts.subs) {
632 s.set_h_position (s.h_position() + (*j)->content->subtitle->x_offset ());
633 s.set_v_position (s.v_position() + (*j)->content->subtitle->y_offset ());
634 float const xs = (*j)->content->subtitle->x_scale();
635 float const ys = (*j)->content->subtitle->y_scale();
636 float size = s.size();
638 /* Adjust size to express the common part of the scaling;
639 e.g. if xs = ys = 0.5 we scale size by 2.
641 if (xs > 1e-5 && ys > 1e-5) {
642 size *= 1 / min (1 / xs, 1 / ys);
646 /* Then express aspect ratio changes */
647 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
648 s.set_aspect_adjust (xs / ys);
650 s.set_in (dcp::Time(content_subtitle_to_dcp (*j, ts.period().from).seconds(), 1000));
651 s.set_out (dcp::Time(content_subtitle_to_dcp (*j, ts.period().to).seconds(), 1000));
652 ps.text.push_back (SubtitleString (s, (*j)->content->subtitle->outline_width()));
653 ps.add_fonts ((*j)->content->subtitle->fonts ());
661 list<shared_ptr<Font> >
662 Player::get_subtitle_fonts ()
664 if (!_have_valid_pieces) {
668 list<shared_ptr<Font> > fonts;
669 BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
670 if (p->content->subtitle) {
671 /* XXX: things may go wrong if there are duplicate font IDs
672 with different font files.
674 list<shared_ptr<Font> > f = p->content->subtitle->fonts ();
675 copy (f.begin(), f.end(), back_inserter (fonts));
682 /** Set this player never to produce any video data */
684 Player::set_ignore_video ()
686 _ignore_video = true;
689 /** Set this player never to produce any audio data */
691 Player::set_ignore_audio ()
693 _ignore_audio = true;
696 /** Set whether or not this player should always burn text subtitles into the image,
697 * regardless of the content settings.
698 * @param burn true to always burn subtitles, false to obey content settings.
701 Player::set_always_burn_subtitles (bool burn)
703 _always_burn_subtitles = burn;
710 _have_valid_pieces = false;
714 Player::set_play_referenced ()
716 _play_referenced = true;
717 _have_valid_pieces = false;
720 list<ReferencedReelAsset>
721 Player::get_reel_assets ()
723 list<ReferencedReelAsset> a;
725 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
726 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
731 scoped_ptr<DCPDecoder> decoder;
733 decoder.reset (new DCPDecoder (j, _film->log()));
739 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
740 DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
741 if (j->reference_video ()) {
743 ReferencedReelAsset (
745 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_picture()->duration(), _film->video_frame_rate()))
750 if (j->reference_audio ()) {
752 ReferencedReelAsset (
754 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_sound()->duration(), _film->video_frame_rate()))
759 if (j->reference_subtitle ()) {
760 DCPOMATIC_ASSERT (k->main_subtitle ());
762 ReferencedReelAsset (
764 DCPTimePeriod (from, from + DCPTime::from_frames (k->main_subtitle()->duration(), _film->video_frame_rate()))
769 /* Assume that main picture duration is the length of the reel */
770 offset += k->main_picture()->duration ();
777 list<shared_ptr<Piece> >
778 Player::overlaps (DCPTime from, DCPTime to, boost::function<bool (Content *)> valid)
780 if (!_have_valid_pieces) {
784 list<shared_ptr<Piece> > overlaps;
785 BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
786 if (valid (i->content.get ()) && i->content->position() < to && i->content->end() > from) {
787 overlaps.push_back (i);
795 Player::reset_pieces ()
797 BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
798 i->decoder->reset ();