2 Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
35 #include "raw_image_proxy.h"
38 #include "render_subtitles.h"
40 #include "content_video.h"
41 #include "player_video.h"
42 #include "frame_rate_change.h"
43 #include "dcp_content.h"
44 #include "dcp_decoder.h"
45 #include "dcp_subtitle_content.h"
46 #include "dcp_subtitle_decoder.h"
47 #include "audio_processor.h"
49 #include <boost/foreach.hpp>
55 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
67 using boost::shared_ptr;
68 using boost::weak_ptr;
69 using boost::dynamic_pointer_cast;
70 using boost::optional;
72 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
74 , _playlist (playlist)
75 , _have_valid_pieces (false)
76 , _ignore_video (false)
77 , _always_burn_subtitles (false)
79 _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
80 _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
81 _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
82 set_video_container_size (_film->frame_size ());
84 film_changed (Film::AUDIO_PROCESSOR);
88 Player::setup_pieces ()
90 list<shared_ptr<Piece> > old_pieces = _pieces;
93 BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
95 if (!i->paths_valid ()) {
99 shared_ptr<Decoder> decoder;
100 optional<FrameRateChange> frc;
102 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
103 DCPTime best_overlap_t;
104 shared_ptr<VideoContent> best_overlap;
105 BOOST_FOREACH (shared_ptr<Content> j, _playlist->content ()) {
106 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (j);
111 DCPTime const overlap = max (vc->position(), i->position()) - min (vc->end(), i->end());
112 if (overlap > best_overlap_t) {
114 best_overlap_t = overlap;
118 optional<FrameRateChange> best_overlap_frc;
120 best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
122 /* No video overlap; e.g. if the DCP is just audio */
123 best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
127 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (i);
129 decoder.reset (new FFmpegDecoder (fc, _film->log()));
130 frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
133 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (i);
135 decoder.reset (new DCPDecoder (dc));
136 frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
140 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (i);
142 /* See if we can re-use an old ImageDecoder */
143 for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
144 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
145 if (imd && imd->content() == ic) {
151 decoder.reset (new ImageDecoder (ic));
154 frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
158 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (i);
160 decoder.reset (new SndfileDecoder (sc));
161 frc = best_overlap_frc;
165 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (i);
167 decoder.reset (new SubRipDecoder (rc));
168 frc = best_overlap_frc;
171 /* DCPSubtitleContent */
172 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (i);
174 decoder.reset (new DCPSubtitleDecoder (dsc));
175 frc = best_overlap_frc;
178 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
179 if (vd && _ignore_video) {
180 vd->set_ignore_video ();
183 _pieces.push_back (shared_ptr<Piece> (new Piece (i, decoder, frc.get ())));
186 _have_valid_pieces = true;
190 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
192 shared_ptr<Content> c = w.lock ();
198 property == ContentProperty::POSITION ||
199 property == ContentProperty::LENGTH ||
200 property == ContentProperty::TRIM_START ||
201 property == ContentProperty::TRIM_END ||
202 property == ContentProperty::PATH ||
203 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
204 property == DCPContentProperty::CAN_BE_PLAYED
207 _have_valid_pieces = false;
211 property == SubtitleContentProperty::USE_SUBTITLES ||
212 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
213 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
214 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
215 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
216 property == SubtitleContentProperty::FONTS ||
217 property == VideoContentProperty::VIDEO_CROP ||
218 property == VideoContentProperty::VIDEO_SCALE ||
219 property == VideoContentProperty::VIDEO_FRAME_RATE ||
220 property == VideoContentProperty::VIDEO_FADE_IN ||
221 property == VideoContentProperty::VIDEO_FADE_OUT
229 Player::set_video_container_size (dcp::Size s)
231 _video_container_size = s;
233 _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
234 _black_image->make_black ();
238 Player::playlist_changed ()
240 _have_valid_pieces = false;
245 Player::film_changed (Film::Property p)
247 /* Here we should notice Film properties that affect our output, and
248 alert listeners that our output now would be different to how it was
249 last time we were run.
252 if (p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
254 } else if (p == Film::AUDIO_PROCESSOR) {
255 if (_film->audio_processor ()) {
256 _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
262 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
264 list<PositionImage> all;
266 for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
271 /* We will scale the subtitle up to fit _video_container_size */
272 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
274 /* Then we need a corrective translation, consisting of two parts:
276 * 1. that which is the result of the scaling of the subtitle by _video_container_size; this will be
277 * rect.x * _video_container_size.width and rect.y * _video_container_size.height.
279 * 2. that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
280 * (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
281 * (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
283 * Combining these two translations gives these expressions.
290 dcp::YUV_TO_RGB_REC601,
291 i->image->pixel_format (),
295 rint (_video_container_size.width * i->rectangle.x),
296 rint (_video_container_size.height * i->rectangle.y)
305 shared_ptr<PlayerVideo>
306 Player::black_player_video_frame (DCPTime time) const
308 return shared_ptr<PlayerVideo> (
310 shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
314 _video_container_size,
315 _video_container_size,
318 PresetColourConversion::all().front().conversion
323 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
324 list<shared_ptr<PlayerVideo> >
325 Player::get_video (DCPTime time, bool accurate)
327 if (!_have_valid_pieces) {
331 /* Find subtitles for possible burn-in */
333 PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false, true);
335 list<PositionImage> sub_images;
337 /* Image subtitles */
338 list<PositionImage> c = transform_image_subtitles (ps.image);
339 copy (c.begin(), c.end(), back_inserter (sub_images));
341 /* Text subtitles (rendered to an image) */
342 if (!ps.text.empty ()) {
343 list<PositionImage> s = render_subtitles (ps.text, ps.fonts, _video_container_size);
344 copy (s.begin (), s.end (), back_inserter (sub_images));
347 optional<PositionImage> subtitles;
348 if (!sub_images.empty ()) {
349 subtitles = merge (sub_images);
352 /* Find pieces containing video which is happening now */
354 list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
356 time + DCPTime::from_frames (1, _film->video_frame_rate ()) - DCPTime::delta()
359 list<shared_ptr<PlayerVideo> > pvf;
362 /* No video content at this time */
363 pvf.push_back (black_player_video_frame (time));
365 /* Some video content at this time */
366 shared_ptr<Piece> last = *(ov.rbegin ());
367 VideoFrameType const last_type = dynamic_pointer_cast<VideoContent> (last->content)->video_frame_type ();
369 /* Get video from appropriate piece(s) */
370 BOOST_FOREACH (shared_ptr<Piece> piece, ov) {
372 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
373 DCPOMATIC_ASSERT (decoder);
374 shared_ptr<VideoContent> video_content = dynamic_pointer_cast<VideoContent> (piece->content);
375 DCPOMATIC_ASSERT (video_content);
378 /* always use the last video */
380 /* with a corresponding L/R eye if appropriate */
381 (last_type == VIDEO_FRAME_TYPE_3D_LEFT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) ||
382 (last_type == VIDEO_FRAME_TYPE_3D_RIGHT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_LEFT);
385 /* We want to use this piece */
386 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
387 if (content_video.empty ()) {
388 pvf.push_back (black_player_video_frame (time));
390 dcp::Size image_size = video_content->scale().size (video_content, _video_container_size, _film->frame_size ());
392 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
394 shared_ptr<PlayerVideo> (
397 content_video_to_dcp (piece, i->frame),
398 video_content->crop (),
399 video_content->fade (i->frame),
401 _video_container_size,
404 video_content->colour_conversion ()
411 /* Discard unused video */
412 decoder->get_video (dcp_to_content_video (piece, time), accurate);
418 BOOST_FOREACH (shared_ptr<PlayerVideo> p, pvf) {
419 p->set_subtitle (subtitles.get ());
426 shared_ptr<AudioBuffers>
427 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
429 if (!_have_valid_pieces) {
433 Frame const length_frames = length.frames_round (_film->audio_frame_rate ());
435 shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
436 audio->make_silent ();
438 list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
443 for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
445 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
446 DCPOMATIC_ASSERT (content);
447 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
448 DCPOMATIC_ASSERT (decoder);
450 /* The time that we should request from the content */
451 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
452 Frame request_frames = length_frames;
454 if (request < DCPTime ()) {
455 /* We went off the start of the content, so we will need to offset
456 the stuff we get back.
459 request_frames += request.frames_round (_film->audio_frame_rate ());
460 if (request_frames < 0) {
463 request = DCPTime ();
466 Frame const content_frame = dcp_to_resampled_audio (*i, request);
468 BOOST_FOREACH (AudioStreamPtr j, content->audio_streams ()) {
470 if (j->channels() == 0) {
471 /* Some content (e.g. DCPs) can have streams with no channels */
475 /* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
476 ContentAudio all = decoder->get_audio (j, content_frame, request_frames, accurate);
479 if (content->audio_gain() != 0) {
480 shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
481 gain->apply_gain (content->audio_gain ());
486 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
487 dcp_mapped->make_silent ();
488 AudioMapping map = j->mapping ();
489 for (int i = 0; i < map.input_channels(); ++i) {
490 for (int j = 0; j < _film->audio_channels(); ++j) {
491 if (map.get (i, j) > 0) {
492 dcp_mapped->accumulate_channel (
502 if (_audio_processor) {
503 dcp_mapped = _audio_processor->run (dcp_mapped);
506 all.audio = dcp_mapped;
508 audio->accumulate_frames (
510 content_frame - all.frame,
511 offset.frames_round (_film->audio_frame_rate()),
512 min (Frame (all.audio->frames()), request_frames)
521 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
523 shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
524 DCPTime s = t - piece->content->position ();
525 s = min (piece->content->length_after_trim(), s);
526 /* We're returning a frame index here so we need to floor() the conversion since we want to know the frame
527 that contains t, I think
529 return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start ()).frames_floor (vc->video_frame_rate ());
533 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
535 shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
536 ContentTime const c = ContentTime::from_frames (f, vc->video_frame_rate ()) - piece->content->trim_start ();
537 return max (DCPTime (), DCPTime (c, piece->frc) + piece->content->position ());
541 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
543 DCPTime s = t - piece->content->position ();
544 s = min (piece->content->length_after_trim(), s);
545 /* See notes in dcp_to_content_video */
546 return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
550 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
552 DCPTime s = t - piece->content->position ();
553 s = min (piece->content->length_after_trim(), s);
554 return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
557 /** @param burnt true to return only subtitles to be burnt, false to return only
558 * subtitles that should not be burnt. This parameter will be ignored if
559 * _always_burn_subtitles is true; in this case, all subtitles will be returned.
562 Player::get_subtitles (DCPTime time, DCPTime length, bool starting, bool burnt)
564 list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
566 PlayerSubtitles ps (time, length);
568 for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
569 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
570 if (!subtitle_content->use_subtitles () || (!_always_burn_subtitles && (burnt != subtitle_content->burn_subtitles ()))) {
574 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
575 ContentTime const from = dcp_to_content_subtitle (*j, time);
576 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
577 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
579 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
580 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
582 /* Apply content's subtitle offsets */
583 i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
584 i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
586 /* Apply content's subtitle scale */
587 i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
588 i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
590 /* Apply a corrective translation to keep the subtitle centred after that scale */
591 i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
592 i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
594 ps.image.push_back (i->sub);
597 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
598 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
599 BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
600 s.set_h_position (s.h_position() + subtitle_content->subtitle_x_offset ());
601 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
602 float const xs = subtitle_content->subtitle_x_scale();
603 float const ys = subtitle_content->subtitle_y_scale();
604 float const average = s.size() * (xs + ys) / 2;
605 s.set_size (average);
606 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
607 s.set_aspect_adjust (xs / ys);
609 ps.text.push_back (s);
610 ps.add_fonts (subtitle_content->fonts ());
618 list<shared_ptr<Font> >
619 Player::get_subtitle_fonts ()
621 if (!_have_valid_pieces) {
625 list<shared_ptr<Font> > fonts;
626 BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
627 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
629 /* XXX: things may go wrong if there are duplicate font IDs
630 with different font files.
632 list<shared_ptr<Font> > f = sc->fonts ();
633 copy (f.begin(), f.end(), back_inserter (fonts));
640 /** Set this player never to produce any video data */
642 Player::set_ignore_video ()
644 _ignore_video = true;
647 /** Set whether or not this player should always burn text subtitles into the image,
648 * regardless of the content settings.
649 * @param burn true to always burn subtitles, false to obey content settings.
652 Player::set_always_burn_subtitles (bool burn)
654 _always_burn_subtitles = burn;