2 Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
35 #include "raw_image_proxy.h"
38 #include "render_subtitles.h"
40 #include "content_video.h"
41 #include "player_video.h"
42 #include "frame_rate_change.h"
43 #include "dcp_content.h"
44 #include "dcp_decoder.h"
45 #include "dcp_subtitle_content.h"
46 #include "dcp_subtitle_decoder.h"
47 #include "audio_processor.h"
49 #include <boost/foreach.hpp>
55 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
67 using boost::shared_ptr;
68 using boost::weak_ptr;
69 using boost::dynamic_pointer_cast;
70 using boost::optional;
72 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
74 , _playlist (playlist)
75 , _have_valid_pieces (false)
76 , _ignore_video (false)
77 , _always_burn_subtitles (false)
79 _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
80 _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
81 _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
82 set_video_container_size (_film->frame_size ());
84 film_changed (Film::AUDIO_PROCESSOR);
88 Player::setup_pieces ()
90 list<shared_ptr<Piece> > old_pieces = _pieces;
93 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
95 if (!i->paths_valid ()) {
99 shared_ptr<Decoder> decoder;
100 optional<FrameRateChange> frc;
102 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
103 DCPTime best_overlap_t;
104 shared_ptr<VideoContent> best_overlap;
105 BOOST_FOREACH (shared_ptr<Content> j, _playlist->content ()) {
106 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (j);
111 DCPTime const overlap = max (vc->position(), i->position()) - min (vc->end(), i->end());
112 if (overlap > best_overlap_t) {
114 best_overlap_t = overlap;
118 optional<FrameRateChange> best_overlap_frc;
120 best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
122 /* No video overlap; e.g. if the DCP is just audio */
123 best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
127 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (i);
129 decoder.reset (new FFmpegDecoder (fc, _film->log()));
130 frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
133 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (i);
135 decoder.reset (new DCPDecoder (dc));
136 frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
140 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (i);
142 /* See if we can re-use an old ImageDecoder */
143 for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
144 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
145 if (imd && imd->content() == ic) {
151 decoder.reset (new ImageDecoder (ic));
154 frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
158 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (i);
160 decoder.reset (new SndfileDecoder (sc));
161 frc = best_overlap_frc;
165 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (i);
167 decoder.reset (new SubRipDecoder (rc));
168 frc = best_overlap_frc;
171 /* DCPSubtitleContent */
172 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (i);
174 decoder.reset (new DCPSubtitleDecoder (dsc));
175 frc = best_overlap_frc;
178 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
179 if (vd && _ignore_video) {
180 vd->set_ignore_video ();
183 _pieces.push_back (shared_ptr<Piece> (new Piece (i, decoder, frc.get ())));
186 _have_valid_pieces = true;
190 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
192 shared_ptr<Content> c = w.lock ();
198 property == ContentProperty::POSITION ||
199 property == ContentProperty::LENGTH ||
200 property == ContentProperty::TRIM_START ||
201 property == ContentProperty::TRIM_END ||
202 property == ContentProperty::PATH ||
203 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
204 property == DCPContentProperty::CAN_BE_PLAYED
207 _have_valid_pieces = false;
211 property == SubtitleContentProperty::USE_SUBTITLES ||
212 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
213 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
214 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
215 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
216 property == SubtitleContentProperty::FONTS ||
217 property == VideoContentProperty::VIDEO_CROP ||
218 property == VideoContentProperty::VIDEO_SCALE ||
219 property == VideoContentProperty::VIDEO_FRAME_RATE ||
220 property == VideoContentProperty::VIDEO_FADE_IN ||
221 property == VideoContentProperty::VIDEO_FADE_OUT ||
222 property == VideoContentProperty::COLOUR_CONVERSION
230 Player::set_video_container_size (dcp::Size s)
232 _video_container_size = s;
234 _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
235 _black_image->make_black ();
239 Player::playlist_changed ()
241 _have_valid_pieces = false;
246 Player::film_changed (Film::Property p)
248 /* Here we should notice Film properties that affect our output, and
249 alert listeners that our output now would be different to how it was
250 last time we were run.
253 if (p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
255 } else if (p == Film::AUDIO_PROCESSOR) {
256 if (_film->audio_processor ()) {
257 _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
263 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
265 list<PositionImage> all;
267 for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
272 /* We will scale the subtitle up to fit _video_container_size */
273 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
275 /* Then we need a corrective translation, consisting of two parts:
277 * 1. that which is the result of the scaling of the subtitle by _video_container_size; this will be
278 * rect.x * _video_container_size.width and rect.y * _video_container_size.height.
280 * 2. that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
281 * (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
282 * (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
284 * Combining these two translations gives these expressions.
291 dcp::YUV_TO_RGB_REC601,
292 i->image->pixel_format (),
296 lrint (_video_container_size.width * i->rectangle.x),
297 lrint (_video_container_size.height * i->rectangle.y)
306 shared_ptr<PlayerVideo>
307 Player::black_player_video_frame (DCPTime time) const
309 return shared_ptr<PlayerVideo> (
311 shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
315 _video_container_size,
316 _video_container_size,
319 PresetColourConversion::all().front().conversion
324 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
325 list<shared_ptr<PlayerVideo> >
326 Player::get_video (DCPTime time, bool accurate)
328 if (!_have_valid_pieces) {
332 /* Find subtitles for possible burn-in */
334 PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false, true);
336 list<PositionImage> sub_images;
338 /* Image subtitles */
339 list<PositionImage> c = transform_image_subtitles (ps.image);
340 copy (c.begin(), c.end(), back_inserter (sub_images));
342 /* Text subtitles (rendered to an image) */
343 if (!ps.text.empty ()) {
344 list<PositionImage> s = render_subtitles (ps.text, ps.fonts, _video_container_size);
345 copy (s.begin (), s.end (), back_inserter (sub_images));
348 optional<PositionImage> subtitles;
349 if (!sub_images.empty ()) {
350 subtitles = merge (sub_images);
353 /* Find pieces containing video which is happening now */
355 list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
357 time + DCPTime::from_frames (1, _film->video_frame_rate ()) - DCPTime::delta()
360 list<shared_ptr<PlayerVideo> > pvf;
363 /* No video content at this time */
364 pvf.push_back (black_player_video_frame (time));
366 /* Some video content at this time */
367 shared_ptr<Piece> last = *(ov.rbegin ());
368 VideoFrameType const last_type = dynamic_pointer_cast<VideoContent> (last->content)->video_frame_type ();
370 /* Get video from appropriate piece(s) */
371 BOOST_FOREACH (shared_ptr<Piece> piece, ov) {
373 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
374 DCPOMATIC_ASSERT (decoder);
375 shared_ptr<VideoContent> video_content = dynamic_pointer_cast<VideoContent> (piece->content);
376 DCPOMATIC_ASSERT (video_content);
379 /* always use the last video */
381 /* with a corresponding L/R eye if appropriate */
382 (last_type == VIDEO_FRAME_TYPE_3D_LEFT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) ||
383 (last_type == VIDEO_FRAME_TYPE_3D_RIGHT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_LEFT);
386 /* We want to use this piece */
387 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
388 if (content_video.empty ()) {
389 pvf.push_back (black_player_video_frame (time));
391 dcp::Size image_size = video_content->scale().size (video_content, _video_container_size, _film->frame_size ());
393 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
395 shared_ptr<PlayerVideo> (
398 content_video_to_dcp (piece, i->frame),
399 video_content->crop (),
400 video_content->fade (i->frame),
402 _video_container_size,
405 video_content->colour_conversion ()
412 /* Discard unused video */
413 decoder->get_video (dcp_to_content_video (piece, time), accurate);
419 BOOST_FOREACH (shared_ptr<PlayerVideo> p, pvf) {
420 p->set_subtitle (subtitles.get ());
427 shared_ptr<AudioBuffers>
428 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
430 if (!_have_valid_pieces) {
434 Frame const length_frames = length.frames_round (_film->audio_frame_rate ());
436 shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
437 audio->make_silent ();
439 list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
444 for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
446 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
447 DCPOMATIC_ASSERT (content);
448 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
449 DCPOMATIC_ASSERT (decoder);
451 /* The time that we should request from the content */
452 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
453 Frame request_frames = length_frames;
455 if (request < DCPTime ()) {
456 /* We went off the start of the content, so we will need to offset
457 the stuff we get back.
460 request_frames += request.frames_round (_film->audio_frame_rate ());
461 if (request_frames < 0) {
464 request = DCPTime ();
467 Frame const content_frame = dcp_to_resampled_audio (*i, request);
469 BOOST_FOREACH (AudioStreamPtr j, content->audio_streams ()) {
471 if (j->channels() == 0) {
472 /* Some content (e.g. DCPs) can have streams with no channels */
476 /* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
477 ContentAudio all = decoder->get_audio (j, content_frame, request_frames, accurate);
480 if (content->audio_gain() != 0) {
481 shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
482 gain->apply_gain (content->audio_gain ());
487 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
488 dcp_mapped->make_silent ();
489 AudioMapping map = j->mapping ();
490 for (int i = 0; i < map.input_channels(); ++i) {
491 for (int j = 0; j < _film->audio_channels(); ++j) {
492 if (map.get (i, j) > 0) {
493 dcp_mapped->accumulate_channel (
503 if (_audio_processor) {
504 dcp_mapped = _audio_processor->run (dcp_mapped);
507 all.audio = dcp_mapped;
509 audio->accumulate_frames (
511 content_frame - all.frame,
512 offset.frames_round (_film->audio_frame_rate()),
513 min (Frame (all.audio->frames()), request_frames)
522 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
524 shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
525 DCPTime s = t - piece->content->position ();
526 s = min (piece->content->length_after_trim(), s);
527 s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
529 /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
530 then convert that ContentTime to frames at the content's rate. However this fails for
531 situations like content at 29.9978733fps, DCP at 30fps. The accuracy of the Time type is not
532 enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
534 Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
536 return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
540 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
542 shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
543 ContentTime const c = ContentTime::from_frames (f, vc->video_frame_rate ()) - piece->content->trim_start ();
544 return max (DCPTime (), DCPTime (c, piece->frc) + piece->content->position ());
548 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
550 DCPTime s = t - piece->content->position ();
551 s = min (piece->content->length_after_trim(), s);
552 /* See notes in dcp_to_content_video */
553 return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
557 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
559 DCPTime s = t - piece->content->position ();
560 s = min (piece->content->length_after_trim(), s);
561 return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
564 /** @param burnt true to return only subtitles to be burnt, false to return only
565 * subtitles that should not be burnt. This parameter will be ignored if
566 * _always_burn_subtitles is true; in this case, all subtitles will be returned.
569 Player::get_subtitles (DCPTime time, DCPTime length, bool starting, bool burnt)
571 list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
573 PlayerSubtitles ps (time, length);
575 for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
576 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
577 if (!subtitle_content->use_subtitles () || (!_always_burn_subtitles && (burnt != subtitle_content->burn_subtitles ()))) {
581 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
582 ContentTime const from = dcp_to_content_subtitle (*j, time);
583 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
584 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
586 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
587 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
589 /* Apply content's subtitle offsets */
590 i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
591 i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
593 /* Apply content's subtitle scale */
594 i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
595 i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
597 /* Apply a corrective translation to keep the subtitle centred after that scale */
598 i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
599 i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
601 ps.image.push_back (i->sub);
604 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
605 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
606 BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
607 s.set_h_position (s.h_position() + subtitle_content->subtitle_x_offset ());
608 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
609 float const xs = subtitle_content->subtitle_x_scale();
610 float const ys = subtitle_content->subtitle_y_scale();
611 float const average = s.size() * (xs + ys) / 2;
612 s.set_size (average);
613 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
614 s.set_aspect_adjust (xs / ys);
616 ps.text.push_back (s);
617 ps.add_fonts (subtitle_content->fonts ());
625 list<shared_ptr<Font> >
626 Player::get_subtitle_fonts ()
628 if (!_have_valid_pieces) {
632 list<shared_ptr<Font> > fonts;
633 BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
634 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
636 /* XXX: things may go wrong if there are duplicate font IDs
637 with different font files.
639 list<shared_ptr<Font> > f = sc->fonts ();
640 copy (f.begin(), f.end(), back_inserter (fonts));
647 /** Set this player never to produce any video data */
649 Player::set_ignore_video ()
651 _ignore_video = true;
654 /** Set whether or not this player should always burn text subtitles into the image,
655 * regardless of the content settings.
656 * @param burn true to always burn subtitles, false to obey content settings.
659 Player::set_always_burn_subtitles (bool burn)
661 _always_burn_subtitles = burn;