X-Git-Url: https://main.carlh.net/gitweb/?a=blobdiff_plain;f=src%2Flib%2Fplayer.cc;h=eeeef2cb3ed4006170127a7d647a186c056d029f;hb=b20c4b11abff0bd57dab03174070ec8571bbe4d5;hp=d6e142c5e810ffd8dd5ec359345d4f35acb2590f;hpb=52dc5e2f2d0e5be8e9ba75b5978a1de58b11cb25;p=dcpomatic.git

diff --git a/src/lib/player.cc b/src/lib/player.cc
index d6e142c5e..eeeef2cb3 100644
--- a/src/lib/player.cc
+++ b/src/lib/player.cc
@@ -1,34 +1,27 @@
 /*
-    Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2013-2017 Carl Hetherington <cth@carlh.net>
 
-    This program is free software; you can redistribute it and/or modify
+    This file is part of DCP-o-matic.
+
+    DCP-o-matic is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
 
-    This program is distributed in the hope that it will be useful,
+    DCP-o-matic is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+    along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
 
 */
 
 #include "player.h"
 #include "film.h"
-#include "ffmpeg_decoder.h"
 #include "audio_buffers.h"
-#include "ffmpeg_content.h"
-#include "image_decoder.h"
-#include "image_content.h"
-#include "sndfile_decoder.h"
-#include "sndfile_content.h"
-#include "subtitle_content.h"
-#include "subrip_decoder.h"
-#include "subrip_content.h"
+#include "content_audio.h"
 #include "dcp_content.h"
 #include "job.h"
 #include "image.h"
@@ -40,19 +33,34 @@
 #include "content_video.h"
 #include "player_video.h"
 #include "frame_rate_change.h"
-#include "dcp_content.h"
-#include "dcp_decoder.h"
-#include "dcp_subtitle_content.h"
-#include "dcp_subtitle_decoder.h"
 #include "audio_processor.h"
 #include "playlist.h"
+#include "referenced_reel_asset.h"
+#include "decoder_factory.h"
+#include "decoder.h"
+#include "video_decoder.h"
+#include "audio_decoder.h"
+#include "subtitle_content.h"
+#include "subtitle_decoder.h"
+#include "ffmpeg_content.h"
+#include "audio_content.h"
+#include "content_subtitle.h"
+#include "dcp_decoder.h"
+#include "image_decoder.h"
+#include "resampler.h"
+#include "compose.hpp"
+#include <dcp/reel.h>
+#include <dcp/reel_sound_asset.h>
+#include <dcp/reel_subtitle_asset.h>
+#include <dcp/reel_picture_asset.h>
 #include <boost/foreach.hpp>
 #include <stdint.h>
 #include <algorithm>
+#include <iostream>
 
 #include "i18n.h"
 
-#define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
+#define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
 
 using std::list;
 using std::cout;
@@ -68,13 +76,18 @@ using boost::shared_ptr;
 using boost::weak_ptr;
 using boost::dynamic_pointer_cast;
 using boost::optional;
+using boost::scoped_ptr;
 
 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
 	: _film (film)
 	, _playlist (playlist)
 	, _have_valid_pieces (false)
 	, _ignore_video (false)
+	, _ignore_audio (false)
 	, _always_burn_subtitles (false)
+	, _fast (false)
+	, _play_referenced (false)
+	, _audio_merger (_film->audio_channels(), _film->audio_frame_rate())
 {
 	_film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
 	_playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
@@ -87,7 +100,6 @@ Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist
 void
 Player::setup_pieces ()
 {
-	list<shared_ptr<Piece> > old_pieces = _pieces;
 	_pieces.clear ();
 
 	BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
@@ -96,91 +108,42 @@ Player::setup_pieces ()
 			continue;
 		}
 
-		shared_ptr<Decoder> decoder;
-		optional<FrameRateChange> frc;
-
-		/* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
-		DCPTime best_overlap_t;
-		shared_ptr<VideoContent> best_overlap;
-		BOOST_FOREACH (shared_ptr<Content> j, _playlist->content ()) {
-			shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (j);
-			if (!vc) {
-				continue;
-			}
+		shared_ptr<Decoder> decoder = decoder_factory (i, _film->log());
+		FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
 
-			DCPTime const overlap = max (vc->position(), i->position()) - min (vc->end(), i->end());
-			if (overlap > best_overlap_t) {
-				best_overlap = vc;
-				best_overlap_t = overlap;
-			}
-		}
-
-		optional<FrameRateChange> best_overlap_frc;
-		if (best_overlap) {
-			best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
-		} else {
-			/* No video overlap; e.g. if the DCP is just audio */
-			best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
+		if (!decoder) {
+			/* Not something that we can decode; e.g. Atmos content */
+			continue;
 		}
 
-		/* FFmpeg */
-		shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (i);
-		if (fc) {
-			decoder.reset (new FFmpegDecoder (fc, _film->log()));
-			frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
+		if (decoder->video && _ignore_video) {
+			decoder->video->set_ignore ();
 		}
 
-		shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (i);
-		if (dc) {
-			decoder.reset (new DCPDecoder (dc));
-			frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
+		if (decoder->audio && _ignore_audio) {
+			decoder->audio->set_ignore ();
 		}
 
-		/* ImageContent */
-		shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (i);
-		if (ic) {
-			/* See if we can re-use an old ImageDecoder */
-			for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
-				shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
-				if (imd && imd->content() == ic) {
-					decoder = imd;
-				}
-			}
-
-			if (!decoder) {
-				decoder.reset (new ImageDecoder (ic));
-			}
-
-			frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
+		shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
+		if (dcp && _play_referenced) {
+			dcp->set_decode_referenced ();
 		}
 
-		/* SndfileContent */
-		shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (i);
-		if (sc) {
-			decoder.reset (new SndfileDecoder (sc));
-			frc = best_overlap_frc;
-		}
+		shared_ptr<Piece> piece (new Piece (i, decoder, frc));
+		_pieces.push_back (piece);
 
-		/* SubRipContent */
-		shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (i);
-		if (rc) {
-			decoder.reset (new SubRipDecoder (rc));
-			frc = best_overlap_frc;
+		if (decoder->video) {
+			decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece> (piece), _1));
 		}
 
-		/* DCPSubtitleContent */
-		shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (i);
-		if (dsc) {
-			decoder.reset (new DCPSubtitleDecoder (dsc));
-			frc = best_overlap_frc;
+		if (decoder->audio) {
+			decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 		}
 
-		shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
-		if (vd && _ignore_video) {
-			vd->set_ignore_video ();
+		if (decoder->subtitle) {
+			decoder->subtitle->ImageData.connect (bind (&Player::image_subtitle, this, weak_ptr<Piece> (piece), _1));
+			decoder->subtitle->TextData.connect (bind (&Player::text_subtitle, this, weak_ptr<Piece> (piece), _1));
 		}
-
-		_pieces.push_back (shared_ptr<Piece> (new Piece (i, decoder, frc.get ())));
 	}
 
 	_have_valid_pieces = true;
@@ -200,26 +163,36 @@ Player::playlist_content_changed (weak_ptr<Content> w, int property, bool freque
 		property == ContentProperty::TRIM_START ||
 		property == ContentProperty::TRIM_END ||
 		property == ContentProperty::PATH ||
-		property == VideoContentProperty::VIDEO_FRAME_TYPE ||
-		property == DCPContentProperty::CAN_BE_PLAYED
+		property == VideoContentProperty::FRAME_TYPE ||
+		property == DCPContentProperty::NEEDS_ASSETS ||
+		property == DCPContentProperty::NEEDS_KDM ||
+		property == SubtitleContentProperty::COLOUR ||
+		property == SubtitleContentProperty::OUTLINE ||
+		property == SubtitleContentProperty::SHADOW ||
+		property == SubtitleContentProperty::EFFECT_COLOUR ||
+		property == FFmpegContentProperty::SUBTITLE_STREAM ||
+		property == VideoContentProperty::COLOUR_CONVERSION
 		) {
 
 		_have_valid_pieces = false;
 		Changed (frequent);
 
 	} else if (
-		property == SubtitleContentProperty::USE_SUBTITLES ||
-		property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
-		property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
-		property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
-		property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
+		property == SubtitleContentProperty::LINE_SPACING ||
+		property == SubtitleContentProperty::OUTLINE_WIDTH ||
+		property == SubtitleContentProperty::Y_SCALE ||
+		property == SubtitleContentProperty::FADE_IN ||
+		property == SubtitleContentProperty::FADE_OUT ||
+		property == ContentProperty::VIDEO_FRAME_RATE ||
+		property == SubtitleContentProperty::USE ||
+		property == SubtitleContentProperty::X_OFFSET ||
+		property == SubtitleContentProperty::Y_OFFSET ||
+		property == SubtitleContentProperty::X_SCALE ||
 		property == SubtitleContentProperty::FONTS ||
-		property == VideoContentProperty::VIDEO_CROP ||
-		property == VideoContentProperty::VIDEO_SCALE ||
-		property == VideoContentProperty::VIDEO_FRAME_RATE ||
-		property == VideoContentProperty::VIDEO_FADE_IN ||
-		property == VideoContentProperty::VIDEO_FADE_OUT ||
-		property == VideoContentProperty::COLOUR_CONVERSION
+		property == VideoContentProperty::CROP ||
+		property == VideoContentProperty::SCALE ||
+		property == VideoContentProperty::FADE_IN ||
+		property == VideoContentProperty::FADE_OUT
 		) {
 
 		Changed (frequent);
@@ -231,7 +204,7 @@ Player::set_video_container_size (dcp::Size s)
 {
 	_video_container_size = s;
 
-	_black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
+	_black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 	_black_image->make_black ();
 }
 
@@ -250,7 +223,13 @@ Player::film_changed (Film::Property p)
 	   last time we were run.
 	*/
 
-	if (p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
+	if (p == Film::CONTAINER) {
+		Changed (false);
+	} else if (p == Film::VIDEO_FRAME_RATE) {
+		/* Pieces contain a FrameRateChange which contains the DCP frame rate,
+		   so we need new pieces here.
+		*/
+		_have_valid_pieces = false;
 		Changed (false);
 	} else if (p == Film::AUDIO_PROCESSOR) {
 		if (_film->audio_processor ()) {
@@ -290,7 +269,8 @@ Player::transform_image_subtitles (list<ImageSubtitle> subs) const
 					scaled_size,
 					dcp::YUV_TO_RGB_REC601,
 					i->image->pixel_format (),
-					true
+					true,
+					_fast
 					),
 				Position<int> (
 					lrint (_video_container_size.width * i->rectangle.x),
@@ -304,12 +284,11 @@ Player::transform_image_subtitles (list<ImageSubtitle> subs) const
 }
 
 shared_ptr<PlayerVideo>
-Player::black_player_video_frame (DCPTime time) const
+Player::black_player_video_frame () const
 {
 	return shared_ptr<PlayerVideo> (
 		new PlayerVideo (
 			shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
-			time,
 			Crop (),
 			optional<double> (),
 			_video_container_size,
@@ -321,342 +300,570 @@ Player::black_player_video_frame (DCPTime time) const
 	);
 }
 
-/** @return All PlayerVideos at the given time (there may be two frames for 3D) */
-list<shared_ptr<PlayerVideo> >
-Player::get_video (DCPTime time, bool accurate)
+Frame
+Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
+{
+	DCPTime s = t - piece->content->position ();
+	s = min (piece->content->length_after_trim(), s);
+	s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
+
+	/* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
+	   then convert that ContentTime to frames at the content's rate.  However this fails for
+	   situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
+	   enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
+
+	   Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
+	*/
+	return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
+}
+
+DCPTime
+Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
+{
+	/* See comment in dcp_to_content_video */
+	DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime (piece->content->trim_start (), piece->frc);
+	return max (DCPTime (), d + piece->content->position ());
+}
+
+Frame
+Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
+{
+	DCPTime s = t - piece->content->position ();
+	s = min (piece->content->length_after_trim(), s);
+	/* See notes in dcp_to_content_video */
+	return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
+}
+
+DCPTime
+Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
+{
+	/* See comment in dcp_to_content_video */
+	DCPTime const d = DCPTime::from_frames (f, _film->audio_frame_rate()) - DCPTime (piece->content->trim_start (), piece->frc);
+	return max (DCPTime (), d + piece->content->position ());
+}
+
+ContentTime
+Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
+{
+	DCPTime s = t - piece->content->position ();
+	s = min (piece->content->length_after_trim(), s);
+	return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
+}
+
+DCPTime
+Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
+{
+	return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
+}
+
+list<shared_ptr<Font> >
+Player::get_subtitle_fonts ()
 {
 	if (!_have_valid_pieces) {
 		setup_pieces ();
 	}
 
-	/* Find subtitles for possible burn-in */
+	list<shared_ptr<Font> > fonts;
+	BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
+		if (p->content->subtitle) {
+			/* XXX: things may go wrong if there are duplicate font IDs
+			   with different font files.
+			*/
+			list<shared_ptr<Font> > f = p->content->subtitle->fonts ();
+			copy (f.begin(), f.end(), back_inserter (fonts));
+		}
+	}
 
-	PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false, true);
+	return fonts;
+}
 
-	list<PositionImage> sub_images;
+/** Set this player never to produce any video data */
+void
+Player::set_ignore_video ()
+{
+	_ignore_video = true;
+}
 
-	/* Image subtitles */
-	list<PositionImage> c = transform_image_subtitles (ps.image);
-	copy (c.begin(), c.end(), back_inserter (sub_images));
+/** Set this player never to produce any audio data */
+void
+Player::set_ignore_audio ()
+{
+	_ignore_audio = true;
+}
 
-	/* Text subtitles (rendered to an image) */
-	if (!ps.text.empty ()) {
-		list<PositionImage> s = render_subtitles (ps.text, ps.fonts, _video_container_size);
-		copy (s.begin (), s.end (), back_inserter (sub_images));
-	}
+/** Set whether or not this player should always burn text subtitles into the image,
+ *  regardless of the content settings.
+ *  @param burn true to always burn subtitles, false to obey content settings.
+ */
+void
+Player::set_always_burn_subtitles (bool burn)
+{
+	_always_burn_subtitles = burn;
+}
 
-	optional<PositionImage> subtitles;
-	if (!sub_images.empty ()) {
-		subtitles = merge (sub_images);
-	}
+void
+Player::set_fast ()
+{
+	_fast = true;
+	_have_valid_pieces = false;
+}
 
-	/* Find pieces containing video which is happening now */
+void
+Player::set_play_referenced ()
+{
+	_play_referenced = true;
+	_have_valid_pieces = false;
+}
 
-	list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
-		time,
-		time + DCPTime::from_frames (1, _film->video_frame_rate ()) - DCPTime::delta()
-		);
+list<ReferencedReelAsset>
+Player::get_reel_assets ()
+{
+	list<ReferencedReelAsset> a;
+
+	BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
+		shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
+		if (!j) {
+			continue;
+		}
 
-	list<shared_ptr<PlayerVideo> > pvf;
+		scoped_ptr<DCPDecoder> decoder;
+		try {
+			decoder.reset (new DCPDecoder (j, _film->log()));
+		} catch (...) {
+			return a;
+		}
 
-	if (ov.empty ()) {
-		/* No video content at this time */
-		pvf.push_back (black_player_video_frame (time));
-	} else {
-		/* Some video content at this time */
-		shared_ptr<Piece> last = *(ov.rbegin ());
-		VideoFrameType const last_type = dynamic_pointer_cast<VideoContent> (last->content)->video_frame_type ();
-
-		/* Get video from appropriate piece(s) */
-		BOOST_FOREACH (shared_ptr<Piece> piece, ov) {
-
-			shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
-			DCPOMATIC_ASSERT (decoder);
-			shared_ptr<VideoContent> video_content = dynamic_pointer_cast<VideoContent> (piece->content);
-			DCPOMATIC_ASSERT (video_content);
-
-			bool const use =
-				/* always use the last video */
-				piece == last ||
-				/* with a corresponding L/R eye if appropriate */
-				(last_type == VIDEO_FRAME_TYPE_3D_LEFT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) ||
-				(last_type == VIDEO_FRAME_TYPE_3D_RIGHT && video_content->video_frame_type() == VIDEO_FRAME_TYPE_3D_LEFT);
-
-			if (use) {
-				/* We want to use this piece */
-				list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
-				if (content_video.empty ()) {
-					pvf.push_back (black_player_video_frame (time));
-				} else {
-					dcp::Size image_size = video_content->scale().size (video_content, _video_container_size, _film->frame_size ());
-
-					for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
-						pvf.push_back (
-							shared_ptr<PlayerVideo> (
-								new PlayerVideo (
-									i->image,
-									content_video_to_dcp (piece, i->frame),
-									video_content->crop (),
-									video_content->fade (i->frame),
-									image_size,
-									_video_container_size,
-									i->eyes,
-									i->part,
-									video_content->colour_conversion ()
-									)
-								)
-							);
-					}
-				}
-			} else {
-				/* Discard unused video */
-				decoder->get_video (dcp_to_content_video (piece, time), accurate);
+		int64_t offset = 0;
+		BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
+
+			DCPOMATIC_ASSERT (j->video_frame_rate ());
+			double const cfr = j->video_frame_rate().get();
+			Frame const trim_start = j->trim_start().frames_round (cfr);
+			Frame const trim_end = j->trim_end().frames_round (cfr);
+			int const ffr = _film->video_frame_rate ();
+
+			DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
+			if (j->reference_video ()) {
+				shared_ptr<dcp::ReelAsset> ra = k->main_picture ();
+				DCPOMATIC_ASSERT (ra);
+				ra->set_entry_point (ra->entry_point() + trim_start);
+				ra->set_duration (ra->duration() - trim_start - trim_end);
+				a.push_back (
+					ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
+					);
 			}
-		}
-	}
 
-	if (subtitles) {
-		BOOST_FOREACH (shared_ptr<PlayerVideo> p, pvf) {
-			p->set_subtitle (subtitles.get ());
+			if (j->reference_audio ()) {
+				shared_ptr<dcp::ReelAsset> ra = k->main_sound ();
+				DCPOMATIC_ASSERT (ra);
+				ra->set_entry_point (ra->entry_point() + trim_start);
+				ra->set_duration (ra->duration() - trim_start - trim_end);
+				a.push_back (
+					ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
+					);
+			}
+
+			if (j->reference_subtitle ()) {
+				shared_ptr<dcp::ReelAsset> ra = k->main_subtitle ();
+				DCPOMATIC_ASSERT (ra);
+				ra->set_entry_point (ra->entry_point() + trim_start);
+				ra->set_duration (ra->duration() - trim_start - trim_end);
+				a.push_back (
+					ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
+					);
+			}
+
+			/* Assume that main picture duration is the length of the reel */
+			offset += k->main_picture()->duration ();
 		}
 	}
 
-	return pvf;
+	return a;
 }
 
-shared_ptr<AudioBuffers>
-Player::get_audio (DCPTime time, DCPTime length, bool accurate)
+list<shared_ptr<Piece> >
+Player::overlaps (DCPTime from, DCPTime to, boost::function<bool (Content *)> valid)
 {
 	if (!_have_valid_pieces) {
 		setup_pieces ();
 	}
 
-	Frame const length_frames = length.frames_round (_film->audio_frame_rate ());
+	list<shared_ptr<Piece> > overlaps;
+	BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
+		if (valid (i->content.get ()) && i->content->position() < to && i->content->end() > from) {
+			overlaps.push_back (i);
+		}
+	}
 
-	shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
-	audio->make_silent ();
+	return overlaps;
+}
 
-	list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
-	if (ov.empty ()) {
-		return audio;
+bool
+Player::pass ()
+{
+	if (!_have_valid_pieces) {
+		setup_pieces ();
 	}
 
-	for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
+	shared_ptr<Piece> earliest;
+	DCPTime earliest_content;
 
-		shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
-		DCPOMATIC_ASSERT (content);
-		shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
-		DCPOMATIC_ASSERT (decoder);
-
-		/* The time that we should request from the content */
-		DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
-		Frame request_frames = length_frames;
-		DCPTime offset;
-		if (request < DCPTime ()) {
-			/* We went off the start of the content, so we will need to offset
-			   the stuff we get back.
-			*/
-			offset = -request;
-			request_frames += request.frames_round (_film->audio_frame_rate ());
-			if (request_frames < 0) {
-				request_frames = 0;
+	BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
+		if (!i->done) {
+			DCPTime const t = i->content->position() + DCPTime (i->decoder->position(), i->frc);
+			if (!earliest || t < earliest_content) {
+				earliest_content = t;
+				earliest = i;
 			}
-			request = DCPTime ();
 		}
+	}
 
-		Frame const content_frame = dcp_to_resampled_audio (*i, request);
+	if (!earliest) {
+		/* No more content; fill up to the length of our playlist with silent black */
 
-		BOOST_FOREACH (AudioStreamPtr j, content->audio_streams ()) {
+		DCPTime const length = _playlist->length ();
 
-			if (j->channels() == 0) {
-				/* Some content (e.g. DCPs) can have streams with no channels */
-				continue;
-			}
+		DCPTime const frame = DCPTime::from_frames (1, _film->video_frame_rate());
+		DCPTime from;
+		if (_last_time) {
+			from = _last_time.get() + frame;
+		}
+		for (DCPTime i = from; i < length; i += frame) {
+			Video (black_player_video_frame (), i);
+		}
 
-			/* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
-			ContentAudio all = decoder->get_audio (j, content_frame, request_frames, accurate);
+		DCPTime t = _last_audio_time;
+		while (t < length) {
+			DCPTime block = min (DCPTime::from_seconds (0.5), length - t);
+			shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), block.frames_round(_film->audio_frame_rate())));
+			silence->make_silent ();
+			Audio (silence, t);
+			t += block;
+		}
 
-			/* Gain */
-			if (content->audio_gain() != 0) {
-				shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
-				gain->apply_gain (content->audio_gain ());
-				all.audio = gain;
-			}
+		return true;
+	}
 
-			/* Remap channels */
-			shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
-			dcp_mapped->make_silent ();
-			AudioMapping map = j->mapping ();
-			for (int i = 0; i < map.input_channels(); ++i) {
-				for (int j = 0; j < _film->audio_channels(); ++j) {
-					if (map.get (i, j) > 0) {
-						dcp_mapped->accumulate_channel (
-							all.audio.get(),
-							i,
-							j,
-							map.get (i, j)
-							);
-					}
-				}
-			}
+	earliest->done = earliest->decoder->pass ();
 
-			if (_audio_processor) {
-				dcp_mapped = _audio_processor->run (dcp_mapped);
-			}
+	/* Emit any audio that is ready */
 
-			all.audio = dcp_mapped;
+	DCPTime pull_from = _playlist->length ();
+	for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
+		if (!i->second.piece->done && i->second.last_push_end < pull_from) {
+			pull_from = i->second.last_push_end;
+		}
+	}
 
-			audio->accumulate_frames (
-				all.audio.get(),
-				content_frame - all.frame,
-				offset.frames_round (_film->audio_frame_rate()),
-				min (Frame (all.audio->frames()), request_frames)
-				);
+	pair<shared_ptr<AudioBuffers>, DCPTime> audio = _audio_merger.pull (pull_from);
+	if (audio.first->frames() > 0) {
+		DCPOMATIC_ASSERT (audio.second >= _last_audio_time);
+		DCPTime t = _last_audio_time;
+		while (t < audio.second) {
+			/* Silence up to the time of this new audio */
+			DCPTime block = min (DCPTime::from_seconds (0.5), audio.second - t);
+			shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), block.frames_round(_film->audio_frame_rate())));
+			silence->make_silent ();
+			Audio (silence, t);
+			t += block;
 		}
+
+		Audio (audio.first, audio.second);
+		_last_audio_time = audio.second + DCPTime::from_frames(audio.first->frames(), _film->audio_frame_rate());
 	}
 
-	return audio;
+	return false;
 }
 
-Frame
-Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
+void
+Player::video (weak_ptr<Piece> wp, ContentVideo video)
 {
-	shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
-	DCPTime s = t - piece->content->position ();
-	s = min (piece->content->length_after_trim(), s);
-	s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
+	shared_ptr<Piece> piece = wp.lock ();
+	if (!piece) {
+		return;
+	}
 
-	/* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
-	   then convert that ContentTime to frames at the content's rate.  However this fails for
-	   situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
-	   enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
+	/* Time and period of the frame we will emit */
+	DCPTime const time = content_video_to_dcp (piece, video.frame);
+	DCPTimePeriod const period (time, time + DCPTime::from_frames (1, _film->video_frame_rate()));
 
-	   Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
-	*/
-	return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
-}
+	/* Discard if it's outside the content's period */
+	if (time < piece->content->position() || time >= piece->content->end()) {
+		return;
+	}
 
-DCPTime
-Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
-{
-	shared_ptr<const VideoContent> vc = dynamic_pointer_cast<const VideoContent> (piece->content);
-	ContentTime const c = ContentTime::from_frames (f, vc->video_frame_rate ()) - piece->content->trim_start ();
-	return max (DCPTime (), DCPTime (c, piece->frc) + piece->content->position ());
-}
+	/* Get any subtitles */
 
-Frame
-Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
-{
-	DCPTime s = t - piece->content->position ();
-	s = min (piece->content->length_after_trim(), s);
-	/* See notes in dcp_to_content_video */
-	return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
-}
+	optional<PositionImage> subtitles;
 
-ContentTime
-Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
-{
-	DCPTime s = t - piece->content->position ();
-	s = min (piece->content->length_after_trim(), s);
-	return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
-}
+	for (list<pair<PlayerSubtitles, DCPTimePeriod> >::const_iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 
-/** @param burnt true to return only subtitles to be burnt, false to return only
- *  subtitles that should not be burnt.  This parameter will be ignored if
- *  _always_burn_subtitles is true; in this case, all subtitles will be returned.
- */
-PlayerSubtitles
-Player::get_subtitles (DCPTime time, DCPTime length, bool starting, bool burnt)
-{
-	list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
+		if (!i->second.overlap (period)) {
+			continue;
+		}
 
-	PlayerSubtitles ps (time, length);
+		list<PositionImage> sub_images;
 
-	for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
-		shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
-		if (!subtitle_content->use_subtitles () || (!_always_burn_subtitles && (burnt != subtitle_content->burn_subtitles ()))) {
-			continue;
+		/* Image subtitles */
+		list<PositionImage> c = transform_image_subtitles (i->first.image);
+		copy (c.begin(), c.end(), back_inserter (sub_images));
+
+		/* Text subtitles (rendered to an image) */
+		if (!i->first.text.empty ()) {
+			list<PositionImage> s = render_subtitles (i->first.text, i->first.fonts, _video_container_size, time);
+			copy (s.begin (), s.end (), back_inserter (sub_images));
+		}
+
+		if (!sub_images.empty ()) {
+			subtitles = merge (sub_images);
+		}
+	}
+
+	/* Fill gaps */
+
+	if (_last_time) {
+		/* XXX: this may not work for 3D */
+		DCPTime const frame = DCPTime::from_frames (1, _film->video_frame_rate());
+		for (DCPTime i = _last_time.get() + frame; i < time; i += frame) {
+			if (_playlist->video_content_at(i) && _last_video) {
+				Video (shared_ptr<PlayerVideo> (new PlayerVideo (*_last_video)), i);
+			} else {
+				Video (black_player_video_frame (), i);
+			}
+		}
+	}
+
+	_last_video.reset (
+		new PlayerVideo (
+			video.image,
+			piece->content->video->crop (),
+			piece->content->video->fade (video.frame),
+			piece->content->video->scale().size (
+				piece->content->video, _video_container_size, _film->frame_size ()
+				),
+			_video_container_size,
+			video.eyes,
+			video.part,
+			piece->content->video->colour_conversion ()
+			)
+		);
+
+	if (subtitles) {
+		_last_video->set_subtitle (subtitles.get ());
+	}
+
+	_last_time = time;
+
+	Video (_last_video, *_last_time);
+
+	/* Discard any subtitles we no longer need */
+
+	for (list<pair<PlayerSubtitles, DCPTimePeriod> >::iterator i = _subtitles.begin (); i != _subtitles.end(); ) {
+		list<pair<PlayerSubtitles, DCPTimePeriod> >::iterator tmp = i;
+		++tmp;
+
+		if (i->second.to < time) {
+			_subtitles.erase (i);
 		}
 
-		shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
-		ContentTime const from = dcp_to_content_subtitle (*j, time);
-		/* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
-		ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
+		i = tmp;
+	}
+}
 
-		list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
-		for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
+void
+Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
+{
+	shared_ptr<Piece> piece = wp.lock ();
+	if (!piece) {
+		return;
+	}
+
+	shared_ptr<AudioContent> content = piece->content->audio;
+	DCPOMATIC_ASSERT (content);
+
+	/* Gain */
+	if (content->gain() != 0) {
+		shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
+		gain->apply_gain (content->gain ());
+		content_audio.audio = gain;
+	}
 
-			/* Apply content's subtitle offsets */
-			i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
-			i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
+	/* Resample */
+	if (stream->frame_rate() != content->resampled_frame_rate()) {
+		shared_ptr<Resampler> r = resampler (content, stream, true);
+		pair<shared_ptr<const AudioBuffers>, Frame> ro = r->run (content_audio.audio, content_audio.frame);
+		content_audio.audio = ro.first;
+		content_audio.frame = ro.second;
+	}
 
-			/* Apply content's subtitle scale */
-			i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
-			i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
+	/* XXX: end-trimming used to be checked here */
 
-			/* Apply a corrective translation to keep the subtitle centred after that scale */
-			i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
-			i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
+	/* Compute time in the DCP */
+	DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame) + DCPTime::from_seconds (content->delay() / 1000.0);
 
-			ps.image.push_back (i->sub);
+	/* Remove anything that comes before the start of the content */
+	if (time < piece->content->position()) {
+		DCPTime const discard_time = piece->content->position() - time;
+		Frame discard_frames = discard_time.frames_round(_film->audio_frame_rate());
+		Frame remaining_frames = content_audio.audio->frames() - discard_frames;
+		if (remaining_frames <= 0) {
+			/* This audio is entirely discarded */
+			return;
 		}
+		shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
+		cut->copy_from (content_audio.audio.get(), remaining_frames, discard_frames, 0);
+		content_audio.audio = cut;
+		time += discard_time;
+	}
 
-		list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
-		BOOST_FOREACH (ContentTextSubtitle& ts, text) {
-			BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
-				s.set_h_position (s.h_position() + subtitle_content->subtitle_x_offset ());
-				s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
-				float const xs = subtitle_content->subtitle_x_scale();
-				float const ys = subtitle_content->subtitle_y_scale();
-				float const average = s.size() * (xs + ys) / 2;
-				s.set_size (average);
-				if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
-					s.set_aspect_adjust (xs / ys);
-				}
-				ps.text.push_back (s);
-				ps.add_fonts (subtitle_content->fonts ());
+	/* Remap channels */
+	shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), content_audio.audio->frames()));
+	dcp_mapped->make_silent ();
+
+	AudioMapping map = stream->mapping ();
+	for (int i = 0; i < map.input_channels(); ++i) {
+		for (int j = 0; j < dcp_mapped->channels(); ++j) {
+			if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
+				dcp_mapped->accumulate_channel (
+					content_audio.audio.get(),
+					i,
+					static_cast<dcp::Channel> (j),
+					map.get (i, static_cast<dcp::Channel> (j))
+					);
 			}
 		}
 	}
 
-	return ps;
+	content_audio.audio = dcp_mapped;
+
+	if (_audio_processor) {
+		content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
+	}
+
+	_audio_merger.push (content_audio.audio, time);
+
+	if (_stream_states.find (stream) == _stream_states.end ()) {
+		_stream_states[stream] = StreamState (piece, time);
+	} else {
+		_stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
+	}
 }
 
-list<shared_ptr<Font> >
-Player::get_subtitle_fonts ()
+void
+Player::image_subtitle (weak_ptr<Piece> wp, ContentImageSubtitle subtitle)
 {
-	if (!_have_valid_pieces) {
-		setup_pieces ();
+	shared_ptr<Piece> piece = wp.lock ();
+	if (!piece) {
+		return;
 	}
 
-	list<shared_ptr<Font> > fonts;
-	BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
-		shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
-		if (sc) {
-			/* XXX: things may go wrong if there are duplicate font IDs
-			   with different font files.
-			*/
-			list<shared_ptr<Font> > f = sc->fonts ();
-			copy (f.begin(), f.end(), back_inserter (fonts));
+	/* Apply content's subtitle offsets */
+	subtitle.sub.rectangle.x += piece->content->subtitle->x_offset ();
+	subtitle.sub.rectangle.y += piece->content->subtitle->y_offset ();
+
+	/* Apply content's subtitle scale */
+	subtitle.sub.rectangle.width *= piece->content->subtitle->x_scale ();
+	subtitle.sub.rectangle.height *= piece->content->subtitle->y_scale ();
+
+	/* Apply a corrective translation to keep the subtitle centred after that scale */
+	subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * (piece->content->subtitle->x_scale() - 1);
+	subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * (piece->content->subtitle->y_scale() - 1);
+
+	PlayerSubtitles ps;
+	ps.image.push_back (subtitle.sub);
+	DCPTimePeriod period (content_time_to_dcp (piece, subtitle.period().from), content_time_to_dcp (piece, subtitle.period().to));
+
+	if (piece->content->subtitle->use() && (piece->content->subtitle->burn() || _always_burn_subtitles)) {
+		_subtitles.push_back (make_pair (ps, period));
+	} else {
+		Subtitle (ps, period);
+	}
+}
+
+void
+Player::text_subtitle (weak_ptr<Piece> wp, ContentTextSubtitle subtitle)
+{
+	shared_ptr<Piece> piece = wp.lock ();
+	if (!piece) {
+		return;
+	}
+
+	PlayerSubtitles ps;
+	DCPTimePeriod const period (content_time_to_dcp (piece, subtitle.period().from), content_time_to_dcp (piece, subtitle.period().to));
+
+	BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
+		s.set_h_position (s.h_position() + piece->content->subtitle->x_offset ());
+		s.set_v_position (s.v_position() + piece->content->subtitle->y_offset ());
+		float const xs = piece->content->subtitle->x_scale();
+		float const ys = piece->content->subtitle->y_scale();
+		float size = s.size();
+
+		/* Adjust size to express the common part of the scaling;
+		   e.g. if xs = ys = 0.5 we scale size by 2.
+		*/
+		if (xs > 1e-5 && ys > 1e-5) {
+			size *= 1 / min (1 / xs, 1 / ys);
+		}
+		s.set_size (size);
+
+		/* Then express aspect ratio changes */
+		if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
+			s.set_aspect_adjust (xs / ys);
 		}
+
+		s.set_in (dcp::Time(period.from.seconds(), 1000));
+		s.set_out (dcp::Time(period.to.seconds(), 1000));
+		ps.text.push_back (SubtitleString (s, piece->content->subtitle->outline_width()));
+		ps.add_fonts (piece->content->subtitle->fonts ());
 	}
 
-	return fonts;
+	if (piece->content->subtitle->use() && (piece->content->subtitle->burn() || _always_burn_subtitles)) {
+		_subtitles.push_back (make_pair (ps, period));
+	} else {
+		Subtitle (ps, period);
+	}
 }
 
-/** Set this player never to produce any video data */
 void
-Player::set_ignore_video ()
+Player::seek (DCPTime time, bool accurate)
 {
-	_ignore_video = true;
+	BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
+		if (i->content->position() <= time && time < i->content->end()) {
+			i->decoder->seek (dcp_to_content_time (i, time), accurate);
+			i->done = false;
+		}
+	}
+
+	if (accurate) {
+		_last_time = time - DCPTime::from_frames (1, _film->video_frame_rate ());
+	} else {
+		_last_time = optional<DCPTime> ();
+	}
 }
 
-/** Set whether or not this player should always burn text subtitles into the image,
- *  regardless of the content settings.
- *  @param burn true to always burn subtitles, false to obey content settings.
- */
-void
-Player::set_always_burn_subtitles (bool burn)
+shared_ptr<Resampler>
+Player::resampler (shared_ptr<const AudioContent> content, AudioStreamPtr stream, bool create)
 {
-	_always_burn_subtitles = burn;
+	ResamplerMap::const_iterator i = _resamplers.find (make_pair (content, stream));
+	if (i != _resamplers.end ()) {
+		return i->second;
+	}
+
+	if (!create) {
+		return shared_ptr<Resampler> ();
+	}
+
+	LOG_GENERAL (
+		"Creating new resampler from %1 to %2 with %3 channels",
+		stream->frame_rate(),
+		content->resampled_frame_rate(),
+		stream->channels()
+		);
+
+	shared_ptr<Resampler> r (
+		new Resampler (stream->frame_rate(), content->resampled_frame_rate(), stream->channels())
+		);
+
+	_resamplers[make_pair(content, stream)] = r;
+	return r;
 }