2 Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
35 #include "raw_image_proxy.h"
38 #include "render_subtitles.h"
40 #include "content_video.h"
41 #include "player_video.h"
42 #include "frame_rate_change.h"
43 #include "dcp_content.h"
44 #include "dcp_decoder.h"
45 #include "dcp_subtitle_content.h"
46 #include "dcp_subtitle_decoder.h"
47 #include "audio_processor.h"
49 #include <boost/foreach.hpp>
55 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
67 using boost::shared_ptr;
68 using boost::weak_ptr;
69 using boost::dynamic_pointer_cast;
70 using boost::optional;
72 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
74 , _playlist (playlist)
75 , _have_valid_pieces (false)
76 , _ignore_video (false)
77 , _ignore_audio (false)
78 , _always_burn_subtitles (false)
81 _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
82 _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
83 _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
84 set_video_container_size (_film->frame_size ());
86 film_changed (Film::AUDIO_PROCESSOR);
90 Player::setup_pieces ()
92 list<shared_ptr<Piece> > old_pieces = _pieces;
95 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
97 if (!i->paths_valid ()) {
101 shared_ptr<Decoder> decoder;
102 optional<FrameRateChange> frc;
104 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
105 DCPTime best_overlap_t;
106 shared_ptr<VideoContent> best_overlap;
107 BOOST_FOREACH (shared_ptr<Content> j, _playlist->content ()) {
108 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (j);
113 DCPTime const overlap = max (vc->position(), i->position()) - min (vc->end(), i->end());
114 if (overlap > best_overlap_t) {
116 best_overlap_t = overlap;
120 optional<FrameRateChange> best_overlap_frc;
122 best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
124 /* No video overlap; e.g. if the DCP is just audio */
125 best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
129 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (i);
131 decoder.reset (new FFmpegDecoder (fc, _film->log(), _fast));
132 frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
135 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (i);
137 decoder.reset (new DCPDecoder (dc, _fast));
138 frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
142 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (i);
144 /* See if we can re-use an old ImageDecoder */
145 for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
146 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
147 if (imd && imd->content() == ic) {
153 decoder.reset (new ImageDecoder (ic));
156 frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
160 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (i);
162 decoder.reset (new SndfileDecoder (sc, _fast));
163 frc = best_overlap_frc;
167 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (i);
169 decoder.reset (new SubRipDecoder (rc));
170 frc = best_overlap_frc;
173 /* DCPSubtitleContent */
174 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (i);
176 decoder.reset (new DCPSubtitleDecoder (dsc));
177 frc = best_overlap_frc;
180 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
181 if (vd && _ignore_video) {
182 vd->set_ignore_video ();
185 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> (decoder);
186 if (ad && _ignore_audio) {
187 ad->set_ignore_audio ();
190 _pieces.push_back (shared_ptr<Piece> (new Piece (i, decoder, frc.get ())));
193 _have_valid_pieces = true;
197 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
199 shared_ptr<Content> c = w.lock ();
205 property == ContentProperty::POSITION ||
206 property == ContentProperty::LENGTH ||
207 property == ContentProperty::TRIM_START ||
208 property == ContentProperty::TRIM_END ||
209 property == ContentProperty::PATH ||
210 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
211 property == DCPContentProperty::CAN_BE_PLAYED
214 _have_valid_pieces = false;
218 property == SubtitleContentProperty::USE_SUBTITLES ||
219 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
220 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
221 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
222 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
223 property == SubtitleContentProperty::FONTS ||
224 property == VideoContentProperty::VIDEO_CROP ||
225 property == VideoContentProperty::VIDEO_SCALE ||
226 property == VideoContentProperty::VIDEO_FRAME_RATE ||
227 property == VideoContentProperty::VIDEO_FADE_IN ||
228 property == VideoContentProperty::VIDEO_FADE_OUT ||
229 property == VideoContentProperty::COLOUR_CONVERSION
237 Player::set_video_container_size (dcp::Size s)
239 _video_container_size = s;
241 _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
242 _black_image->make_black ();
246 Player::playlist_changed ()
248 _have_valid_pieces = false;
253 Player::film_changed (Film::Property p)
255 /* Here we should notice Film properties that affect our output, and
256 alert listeners that our output now would be different to how it was
257 last time we were run.
260 if (p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
262 } else if (p == Film::AUDIO_PROCESSOR) {
263 if (_film->audio_processor ()) {
264 _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
270 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
272 list<PositionImage> all;
274 for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
279 /* We will scale the subtitle up to fit _video_container_size */
280 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
282 /* Then we need a corrective translation, consisting of two parts:
284 * 1. that which is the result of the scaling of the subtitle by _video_container_size; this will be
285 * rect.x * _video_container_size.width and rect.y * _video_container_size.height.
287 * 2. that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
288 * (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
289 * (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
291 * Combining these two translations gives these expressions.
298 dcp::YUV_TO_RGB_REC601,
299 i->image->pixel_format (),
303 lrint (_video_container_size.width * i->rectangle.x),
304 lrint (_video_container_size.height * i->rectangle.y)
313 shared_ptr<PlayerVideo>
314 Player::black_player_video_frame (DCPTime time) const
316 return shared_ptr<PlayerVideo> (
318 shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
322 _video_container_size,
323 _video_container_size,
326 PresetColourConversion::all().front().conversion
331 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
332 list<shared_ptr<PlayerVideo> >
333 Player::get_video (DCPTime time, bool accurate)
335 if (!_have_valid_pieces) {
339 /* Find subtitles for possible burn-in */
341 PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false, true);
343 list<PositionImage> sub_images;
345 /* Image subtitles */
346 list<PositionImage> c = transform_image_subtitles (ps.image);
347 copy (c.begin(), c.end(), back_inserter (sub_images));
349 /* Text subtitles (rendered to an image) */
350 if (!ps.text.empty ()) {
351 list<PositionImage> s = render_subtitles (ps.text, ps.fonts, _video_container_size);
352 copy (s.begin (), s.end (), back_inserter (sub_images));
355 optional<PositionImage> subtitles;
356 if (!sub_images.empty ()) {
357 subtitles = merge (sub_images);
360 /* Find pieces containing video which is happening now */
362 list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
364 time + DCPTime::from_frames (1, _film->video_frame_rate ()) - DCPTime::delta()
367 list<shared_ptr<PlayerVideo> > pvf;
370 /* No video content at this time */
371 pvf.push_back (black_player_video_frame (time));
373 /* Some video content at this time */
374 shared_ptr<Piece> last = *(ov.rbegin ());
375 VideoFrameType const last_type = dynamic_pointer_cast<VideoContent> (last->content)->video_frame_type ();
377 /* Get video from appropriate piece(s) */
378 BOOST_FOREACH (shared_ptr<Piece> piece, ov) {
380 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
381 DCPOMATIC_ASSERT (decoder);
382 shared_ptr<VideoContent> video_content = dynamic_pointer_cast<VideoContent> (piece->content);
383 DCPOMATIC_ASSERT (video_content);
386 /* always use the last video */
388 /* with a corresponding L/R eye if appropriate */
389 (last_type == VIDEO_FRAME_TYPE_3D_LEFT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) ||
390 (last_type == VIDEO_FRAME_TYPE_3D_RIGHT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_LEFT);
393 /* We want to use this piece */
394 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
395 if (content_video.empty ()) {
396 pvf.push_back (black_player_video_frame (time));
398 dcp::Size image_size = video_content->scale().size (video_content, _video_container_size, _film->frame_size ());
400 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
402 shared_ptr<PlayerVideo> (
405 content_video_to_dcp (piece, i->frame),
406 video_content->crop (),
407 video_content->fade (i->frame),
409 _video_container_size,
412 video_content->colour_conversion ()
419 /* Discard unused video */
420 decoder->get_video (dcp_to_content_video (piece, time), accurate);
426 BOOST_FOREACH (shared_ptr<PlayerVideo> p, pvf) {
427 p->set_subtitle (subtitles.get ());
434 shared_ptr<AudioBuffers>
435 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
437 if (!_have_valid_pieces) {
441 Frame const length_frames = length.frames_round (_film->audio_frame_rate ());
443 shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
444 audio->make_silent ();
446 list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
451 for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
453 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
454 DCPOMATIC_ASSERT (content);
455 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
456 DCPOMATIC_ASSERT (decoder);
458 /* The time that we should request from the content */
459 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
460 Frame request_frames = length_frames;
462 if (request < DCPTime ()) {
463 /* We went off the start of the content, so we will need to offset
464 the stuff we get back.
467 request_frames += request.frames_round (_film->audio_frame_rate ());
468 if (request_frames < 0) {
471 request = DCPTime ();
474 Frame const content_frame = dcp_to_resampled_audio (*i, request);
476 BOOST_FOREACH (AudioStreamPtr j, content->audio_streams ()) {
478 if (j->channels() == 0) {
479 /* Some content (e.g. DCPs) can have streams with no channels */
483 /* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
484 ContentAudio all = decoder->get_audio (j, content_frame, request_frames, accurate);
487 if (content->audio_gain() != 0) {
488 shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
489 gain->apply_gain (content->audio_gain ());
494 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
495 dcp_mapped->make_silent ();
496 AudioMapping map = j->mapping ();
497 for (int i = 0; i < map.input_channels(); ++i) {
498 for (int j = 0; j < _film->audio_channels(); ++j) {
499 if (map.get (i, j) > 0) {
500 dcp_mapped->accumulate_channel (
510 if (_audio_processor) {
511 dcp_mapped = _audio_processor->run (dcp_mapped, _film->audio_channels ());
514 all.audio = dcp_mapped;
516 audio->accumulate_frames (
518 content_frame - all.frame,
519 offset.frames_round (_film->audio_frame_rate()),
520 min (Frame (all.audio->frames()), request_frames)
529 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
531 shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
532 DCPTime s = t - piece->content->position ();
533 s = min (piece->content->length_after_trim(), s);
534 s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
536 /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
537 then convert that ContentTime to frames at the content's rate. However this fails for
538 situations like content at 29.9978733fps, DCP at 30fps. The accuracy of the Time type is not
539 enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
541 Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
543 return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
547 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
549 shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
550 /* See comment in dcp_to_content_video */
551 DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime (piece->content->trim_start (), piece->frc);
552 return max (DCPTime (), d + piece->content->position ());
556 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
558 DCPTime s = t - piece->content->position ();
559 s = min (piece->content->length_after_trim(), s);
560 /* See notes in dcp_to_content_video */
561 return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
565 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
567 DCPTime s = t - piece->content->position ();
568 s = min (piece->content->length_after_trim(), s);
569 return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
572 /** @param burnt true to return only subtitles to be burnt, false to return only
573 * subtitles that should not be burnt. This parameter will be ignored if
574 * _always_burn_subtitles is true; in this case, all subtitles will be returned.
577 Player::get_subtitles (DCPTime time, DCPTime length, bool starting, bool burnt)
579 list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
581 PlayerSubtitles ps (time, length);
583 for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
584 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
585 if (!subtitle_content->use_subtitles () || (!_always_burn_subtitles && (burnt != subtitle_content->burn_subtitles ()))) {
589 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
590 ContentTime const from = dcp_to_content_subtitle (*j, time);
591 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
592 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
594 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
595 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
597 /* Apply content's subtitle offsets */
598 i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
599 i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
601 /* Apply content's subtitle scale */
602 i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
603 i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
605 /* Apply a corrective translation to keep the subtitle centred after that scale */
606 i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
607 i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
609 ps.image.push_back (i->sub);
612 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
613 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
614 BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
615 s.set_h_position (s.h_position() + subtitle_content->subtitle_x_offset ());
616 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
617 float const xs = subtitle_content->subtitle_x_scale();
618 float const ys = subtitle_content->subtitle_y_scale();
619 float const average = s.size() * (xs + ys) / 2;
620 s.set_size (average);
621 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
622 s.set_aspect_adjust (xs / ys);
624 ps.text.push_back (s);
625 ps.add_fonts (subtitle_content->fonts ());
633 list<shared_ptr<Font> >
634 Player::get_subtitle_fonts ()
636 if (!_have_valid_pieces) {
640 list<shared_ptr<Font> > fonts;
641 BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
642 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
644 /* XXX: things may go wrong if there are duplicate font IDs
645 with different font files.
647 list<shared_ptr<Font> > f = sc->fonts ();
648 copy (f.begin(), f.end(), back_inserter (fonts));
655 /** Set this player never to produce any video data */
657 Player::set_ignore_video ()
659 _ignore_video = true;
662 /** Set this player never to produce any audio data */
664 Player::set_ignore_audio ()
666 _ignore_audio = true;
669 /** Set whether or not this player should always burn text subtitles into the image,
670 * regardless of the content settings.
671 * @param burn true to always burn subtitles, false to obey content settings.
674 Player::set_always_burn_subtitles (bool burn)
676 _always_burn_subtitles = burn;
683 _have_valid_pieces = false;