From c719feb3cc4a5627827f8576a9b34756f701d59d Mon Sep 17 00:00:00 2001
From: Carl Hetherington <cth@carlh.net>
Date: Sat, 13 Jul 2013 11:01:10 +0100
Subject: [PATCH] Various fixes to PTS mangling.

---
 src/lib/ffmpeg_content.cc  |  2 +-
 src/lib/ffmpeg_content.h   | 12 +++++-
 src/lib/ffmpeg_decoder.cc  | 78 +++++++++++++++++++++++++-------------
 src/lib/ffmpeg_decoder.h   |  3 +-
 src/lib/filter_graph.cc    |  8 ++--
 src/lib/filter_graph.h     |  2 +-
 src/lib/subtitle_content.h |  4 +-
 src/lib/video_content.h    |  2 +
 test/ffmpeg_pts_offset.cc  | 58 ++++++++++++++++++++++------
 9 files changed, 122 insertions(+), 47 deletions(-)

diff --git a/src/lib/ffmpeg_content.cc b/src/lib/ffmpeg_content.cc
index 88fc6dbd2..f4e1b9e72 100644
--- a/src/lib/ffmpeg_content.cc
+++ b/src/lib/ffmpeg_content.cc
@@ -79,7 +79,7 @@ FFmpegContent::FFmpegContent (shared_ptr<const Film> f, shared_ptr<const cxml::N
 		_filters.push_back (Filter::from_id ((*i)->content ()));
 	}
 
-	_first_video = node->optional_number_child<Time> ("FirstVideo");
+	_first_video = node->optional_number_child<double> ("FirstVideo");
 }
 
 FFmpegContent::FFmpegContent (FFmpegContent const & o)
diff --git a/src/lib/ffmpeg_content.h b/src/lib/ffmpeg_content.h
index 69b459fb6..dba06990b 100644
--- a/src/lib/ffmpeg_content.h
+++ b/src/lib/ffmpeg_content.h
@@ -26,6 +26,7 @@
 #include "subtitle_content.h"
 
 class Filter;
+class ffmpeg_pts_offset_test;
 
 class FFmpegAudioStream
 {
@@ -48,6 +49,11 @@ public:
 	int channels;
 	AudioMapping mapping;
 	boost::optional<double> first_audio;
+
+private:
+	friend class ffmpeg_pts_offset_test;
+	/* Constructor for tests */
+	FFmpegAudioStream () {}
 };
 
 extern bool operator== (FFmpegAudioStream const & a, FFmpegAudioStream const & b);
@@ -138,17 +144,19 @@ public:
 	void set_subtitle_stream (boost::shared_ptr<FFmpegSubtitleStream>);
 	void set_audio_stream (boost::shared_ptr<FFmpegAudioStream>);
 
-	boost::optional<Time> first_video () const {
+	boost::optional<double> first_video () const {
 		boost::mutex::scoped_lock lm (_mutex);
 		return _first_video;
 	}
 	
 private:
+	friend class ffmpeg_pts_offset_test;
+	
 	std::vector<boost::shared_ptr<FFmpegSubtitleStream> > _subtitle_streams;
 	boost::shared_ptr<FFmpegSubtitleStream> _subtitle_stream;
 	std::vector<boost::shared_ptr<FFmpegAudioStream> > _audio_streams;
 	boost::shared_ptr<FFmpegAudioStream> _audio_stream;
-	boost::optional<Time> _first_video;
+	boost::optional<double> _first_video;
 	/** Video filters that should be used when generating DCPs */
 	std::vector<Filter const *> _filters;
 };
diff --git a/src/lib/ffmpeg_decoder.cc b/src/lib/ffmpeg_decoder.cc
index c77cb71c0..201fac8aa 100644
--- a/src/lib/ffmpeg_decoder.cc
+++ b/src/lib/ffmpeg_decoder.cc
@@ -51,6 +51,7 @@ using std::vector;
 using std::stringstream;
 using std::list;
 using std::min;
+using std::pair;
 using boost::shared_ptr;
 using boost::optional;
 using boost::dynamic_pointer_cast;
@@ -66,28 +67,50 @@ FFmpegDecoder::FFmpegDecoder (shared_ptr<const Film> f, shared_ptr<const FFmpegC
 	, _subtitle_codec (0)
 	, _decode_video (video)
 	, _decode_audio (audio)
-	, _pts_offset (0)
+	, _video_pts_offset (0)
+	, _audio_pts_offset (0)
 	, _just_sought (false)
 {
 	setup_subtitle ();
 
-	if (video && audio && c->audio_stream() && c->first_video() && c->audio_stream()->first_audio) {
-		_pts_offset = compute_pts_offset (c->first_video().get(), c->audio_stream()->first_audio.get(), c->video_frame_rate());
-	}
-}
+	/* Audio and video frame PTS values may not start with 0.  We want
+	   to fiddle them so that:
 
-double
-FFmpegDecoder::compute_pts_offset (double first_video, double first_audio, float video_frame_rate)
-{
-	double const old_first_video = first_video;
-	
-	/* Round the first video to a frame boundary */
-	if (fabs (rint (first_video * video_frame_rate) - first_video * video_frame_rate) > 1e-6) {
-		first_video = ceil (first_video * video_frame_rate) / video_frame_rate;
+	   1.  One of them starts at time 0.
+	   2.  The first video PTS value ends up on a frame boundary.
+
+	   Then we remove big initial gaps in PTS and we allow our
+	   insertion of black frames to work.
+
+	   We will do:
+	     audio_pts_to_use = audio_pts_from_ffmpeg + audio_pts_offset;
+	     video_pts_to_use = video_pts_from_ffmpeg + video_pts_offset;
+	*/
+
+	bool const have_video = video && c->first_video();
+	bool const have_audio = audio && c->audio_stream() && c->audio_stream()->first_audio;
+
+	/* First, make one of them start at 0 */
+
+	if (have_audio && have_video) {
+		_video_pts_offset = _audio_pts_offset = - min (c->first_video().get(), c->audio_stream()->first_audio.get());
+	} else if (have_video) {
+		_video_pts_offset = - c->first_video().get();
 	}
 
-	/* Compute the required offset (also removing any common start delay) */
-	return first_video - old_first_video - min (first_video, first_audio);
+	/* Now adjust both so that the video pts starts on a frame */
+	if (have_video && have_audio) {
+		double first_video = c->first_video().get() + _video_pts_offset;
+		double const old_first_video = first_video;
+		
+		/* Round the first video up to a frame boundary */
+		if (fabs (rint (first_video * c->video_frame_rate()) - first_video * c->video_frame_rate()) > 1e-6) {
+			first_video = ceil (first_video * c->video_frame_rate()) / c->video_frame_rate ();
+		}
+
+		_video_pts_offset += first_video - old_first_video;
+		_audio_pts_offset += first_video - old_first_video;
+	}
 }
 
 FFmpegDecoder::~FFmpegDecoder ()
@@ -274,11 +297,15 @@ FFmpegDecoder::seek (VideoContent::Frame frame, bool accurate)
 		initial -= 5;
 	}
 
+	if (initial < 0) {
+		initial = 0;
+	}
+
 	/* Initial seek time in the stream's timebase */
-	int64_t const initial_vt = initial / (_ffmpeg_content->video_frame_rate() * time_base);
+	int64_t const initial_vt = ((initial / _ffmpeg_content->video_frame_rate()) - _video_pts_offset) / time_base;
 	/* Wanted final seek time in the stream's timebase */
-	int64_t const final_vt = frame / (_ffmpeg_content->video_frame_rate() * time_base);
-	
+	int64_t const final_vt = ((frame / _ffmpeg_content->video_frame_rate()) - _video_pts_offset) / time_base;
+
 	av_seek_frame (_format_context, _video_stream, initial_vt, AVSEEK_FLAG_BACKWARD);
 
 	avcodec_flush_buffers (video_codec_context());
@@ -309,7 +336,7 @@ FFmpegDecoder::seek (VideoContent::Frame frame, bool accurate)
 					int64_t const bet = av_frame_get_best_effort_timestamp (_frame);
 					if (bet >= final_vt) {
 						_video_position = rint (
-							(bet * time_base + _pts_offset)	* _ffmpeg_content->video_frame_rate()
+							(bet * time_base + _video_pts_offset) * _ffmpeg_content->video_frame_rate()
 							);
 						av_free_packet (&_packet);
 						break;
@@ -341,7 +368,7 @@ FFmpegDecoder::decode_audio_packet ()
 				if (_audio_position == 0) {
 					/* Where we are in the source, in seconds */
 					double const pts = av_q2d (_format_context->streams[copy_packet.stream_index]->time_base)
-						* av_frame_get_best_effort_timestamp(_frame) - _pts_offset;
+						* av_frame_get_best_effort_timestamp(_frame) + _audio_pts_offset;
 
 					if (pts > 0) {
 						/* Emit some silence */
@@ -393,21 +420,20 @@ FFmpegDecoder::decode_video_packet ()
 		graph = *i;
 	}
 
-	list<shared_ptr<Image> > images = graph->process (_frame);
+	list<pair<shared_ptr<Image>, int64_t> > images = graph->process (_frame);
 
 	string post_process = Filter::ffmpeg_strings (_ffmpeg_content->filters()).second;
 	
-	for (list<shared_ptr<Image> >::iterator i = images.begin(); i != images.end(); ++i) {
+	for (list<pair<shared_ptr<Image>, int64_t> >::iterator i = images.begin(); i != images.end(); ++i) {
 
-		shared_ptr<Image> image = *i;
+		shared_ptr<Image> image = i->first;
 		if (!post_process.empty ()) {
 			image = image->post_process (post_process, true);
 		}
 		
-		int64_t const bet = av_frame_get_best_effort_timestamp (_frame);
-		if (bet != AV_NOPTS_VALUE) {
+		if (i->second != AV_NOPTS_VALUE) {
 
-			double const pts = bet * av_q2d (_format_context->streams[_video_stream]->time_base) - _pts_offset;
+			double const pts = i->second * av_q2d (_format_context->streams[_video_stream]->time_base) + _video_pts_offset;
 
 			if (_just_sought) {
 				/* We just did a seek, so disable any attempts to correct for where we
diff --git a/src/lib/ffmpeg_decoder.h b/src/lib/ffmpeg_decoder.h
index 8819954db..73edcccb4 100644
--- a/src/lib/ffmpeg_decoder.h
+++ b/src/lib/ffmpeg_decoder.h
@@ -84,6 +84,7 @@ private:
 	bool _decode_video;
 	bool _decode_audio;
 
-	double _pts_offset;
+	double _video_pts_offset;
+	double _audio_pts_offset;
 	bool _just_sought;
 };
diff --git a/src/lib/filter_graph.cc b/src/lib/filter_graph.cc
index 3366f8d1b..7c4df8903 100644
--- a/src/lib/filter_graph.cc
+++ b/src/lib/filter_graph.cc
@@ -39,6 +39,8 @@ extern "C" {
 using std::stringstream;
 using std::string;
 using std::list;
+using std::pair;
+using std::make_pair;
 using std::cout;
 using boost::shared_ptr;
 using boost::weak_ptr;
@@ -132,10 +134,10 @@ FilterGraph::~FilterGraph ()
 /** Take an AVFrame and process it using our configured filters, returning a
  *  set of Images.  Caller handles memory management of the input frame.
  */
-list<shared_ptr<Image> >
+list<pair<shared_ptr<Image>, int64_t> >
 FilterGraph::process (AVFrame* frame)
 {
-	list<shared_ptr<Image> > images;
+	list<pair<shared_ptr<Image>, int64_t> > images;
 
 	if (av_buffersrc_write_frame (_buffer_src_context, frame) < 0) {
 		throw DecodeError (N_("could not push buffer into filter chain."));
@@ -146,7 +148,7 @@ FilterGraph::process (AVFrame* frame)
 			break;
 		}
 
-		images.push_back (shared_ptr<Image> (new Image (_frame)));
+		images.push_back (make_pair (shared_ptr<Image> (new Image (_frame)), av_frame_get_best_effort_timestamp (_frame)));
 		av_frame_unref (_frame);
 	}
 	
diff --git a/src/lib/filter_graph.h b/src/lib/filter_graph.h
index e294812c2..494290795 100644
--- a/src/lib/filter_graph.h
+++ b/src/lib/filter_graph.h
@@ -39,7 +39,7 @@ public:
 	~FilterGraph ();
 
 	bool can_process (libdcp::Size s, AVPixelFormat p) const;
-	std::list<boost::shared_ptr<Image> > process (AVFrame * frame);
+	std::list<std::pair<boost::shared_ptr<Image>, int64_t> > process (AVFrame * frame);
 
 private:
 	AVFilterContext* _buffer_src_context;
diff --git a/src/lib/subtitle_content.h b/src/lib/subtitle_content.h
index 1092b7b1c..c29485fee 100644
--- a/src/lib/subtitle_content.h
+++ b/src/lib/subtitle_content.h
@@ -50,7 +50,9 @@ public:
 		return _subtitle_scale;
 	}
 	
-private:	
+private:
+	friend class ffmpeg_pts_offset_test;
+	
 	/** y offset for placing subtitles, as a proportion of the container height;
 	    +ve is further down the frame, -ve is further up.
 	*/
diff --git a/src/lib/video_content.h b/src/lib/video_content.h
index e7bbad9e1..46c8efdeb 100644
--- a/src/lib/video_content.h
+++ b/src/lib/video_content.h
@@ -87,6 +87,8 @@ protected:
 	VideoContent::Frame _video_length;
 
 private:
+	friend class ffmpeg_pts_offset_test;
+	
 	libdcp::Size _video_size;
 	float _video_frame_rate;
 	Crop _crop;
diff --git a/test/ffmpeg_pts_offset.cc b/test/ffmpeg_pts_offset.cc
index 15c2b38e1..0ed8f66d8 100644
--- a/test/ffmpeg_pts_offset.cc
+++ b/test/ffmpeg_pts_offset.cc
@@ -19,21 +19,55 @@
 
 BOOST_AUTO_TEST_CASE (ffmpeg_pts_offset_test)
 {
-	/* Sound == video so no offset required */
-	BOOST_CHECK_EQUAL (FFmpegDecoder::compute_pts_offset (0, 0, 24), 0);
+	shared_ptr<Film> film = new_test_film ("ffmpeg_pts_offset_test");
+	shared_ptr<FFmpegContent> content (new FFmpegContent (film, "test/data/test.mp4"));
+	content->_audio_stream.reset (new FFmpegAudioStream);
+	content->_video_frame_rate = 24;
 
-	/* Common offset should be removed */
-	BOOST_CHECK_CLOSE (FFmpegDecoder::compute_pts_offset (42, 42, 24), -42, 1e-9);
+	{
+		/* Sound == video so no offset required */
+		content->_first_video = 0;
+		content->_audio_stream->first_audio = 0;
+		FFmpegDecoder decoder (film, content, true, true);
+		BOOST_CHECK_EQUAL (decoder._video_pts_offset, 0);
+		BOOST_CHECK_EQUAL (decoder._audio_pts_offset, 0);
+	}
 
-	/* Video is on a frame boundary */
-	BOOST_CHECK_EQUAL (FFmpegDecoder::compute_pts_offset (1.0 / 24.0, 0, 24), 0);
+	{
+		/* Common offset should be removed */
+		content->_first_video = 600;
+		content->_audio_stream->first_audio = 600;
+		FFmpegDecoder decoder (film, content, true, true);
+		BOOST_CHECK_EQUAL (decoder._video_pts_offset, -600);
+		BOOST_CHECK_EQUAL (decoder._audio_pts_offset, -600);
+	}
 
-	/* Again, video is on a frame boundary */
-	BOOST_CHECK_EQUAL (FFmpegDecoder::compute_pts_offset (1.0 / 23.97, 0, 23.97), 0);
+	{
+		/* Video is on a frame boundary */
+		content->_first_video = 1.0 / 24.0;
+		content->_audio_stream->first_audio = 0;
+		FFmpegDecoder decoder (film, content, true, true);
+		BOOST_CHECK_EQUAL (decoder._video_pts_offset, 0);
+		BOOST_CHECK_EQUAL (decoder._audio_pts_offset, 0);
+	}
 
-	/* And again, video is on a frame boundary */
-	BOOST_CHECK_EQUAL (FFmpegDecoder::compute_pts_offset (3.0 / 23.97, 0, 23.97), 0);
+	{
+		/* Video is off a frame boundary */
+		double const frame = 1.0 / 24.0;
+		content->_first_video = frame + 0.0215;
+		content->_audio_stream->first_audio = 0;
+		FFmpegDecoder decoder (film, content, true, true);
+		BOOST_CHECK_EQUAL (decoder._video_pts_offset, (frame - 0.0215));
+		BOOST_CHECK_EQUAL (decoder._audio_pts_offset, (frame - 0.0215));
+	}
 
-	/* Off a frame boundary */
-	BOOST_CHECK_CLOSE (FFmpegDecoder::compute_pts_offset (1.0 / 24.0 - 0.0215, 0, 24), 0.0215, 1e-9);
+	{
+		/* Video is off a frame boundary and both have a common offset */
+		double const frame = 1.0 / 24.0;
+		content->_first_video = frame + 0.0215 + 4.1;
+		content->_audio_stream->first_audio = 4.1;
+		FFmpegDecoder decoder (film, content, true, true);
+		BOOST_CHECK_EQUAL (decoder._video_pts_offset, (frame - 0.0215) - 4.1);
+		BOOST_CHECK_EQUAL (decoder._audio_pts_offset, (frame - 0.0215) - 4.1);
+	}
 }
-- 
2.30.2