#include "exceptions.h"
#include "log.h"
#include "resampler.h"
+#include "util.h"
+#include "film.h"
#include "i18n.h"
AudioDecoder::AudioDecoder (shared_ptr<const Film> film, shared_ptr<const AudioContent> content)
: Decoder (film)
, _audio_content (content)
- , _last_audio (0)
+ , _audio_position (0)
{
if (content->output_audio_frame_rate() != content->content_audio_frame_rate() && content->audio_channels ()) {
_resampler.reset (new Resampler (content->content_audio_frame_rate(), content->output_audio_frame_rate(), content->audio_channels ()));
}
}
+/** Audio timestamping is made hard by many factors, but the final nail in the coffin is resampling.
+ * We have to assume that we are feeding continuous data into the resampler, and so we get continuous
+ * data out. Hence we do the timestamping here, post-resampler, just by counting samples.
+ */
void
-AudioDecoder::audio (shared_ptr<const AudioBuffers> data, ContentTime time)
+AudioDecoder::audio (shared_ptr<const AudioBuffers> data)
{
if (_resampler) {
data = _resampler->run (data);
}
- _pending.push_back (shared_ptr<DecodedAudio> (new DecodedAudio (data, time)));
- _last_audio = time + (data->frames() * TIME_HZ / _audio_content->output_audio_frame_rate());
+ _pending.push_back (shared_ptr<DecodedAudio> (new DecodedAudio (data, _audio_position)));
+ _audio_position += data->frames ();
}
void
shared_ptr<const AudioBuffers> b = _resampler->flush ();
if (b) {
- audio (b, _last_audio);
+ _pending.push_back (shared_ptr<DecodedAudio> (new DecodedAudio (b, _audio_position)));
+ _audio_position += b->frames ();
}
}
+
+void
+AudioDecoder::seek (ContentTime t, bool)
+{
+ shared_ptr<const Film> film = _film.lock ();
+ assert (film);
+
+ FrameRateChange frc = film->active_frame_rate_change (_audio_content->position ());
+ _audio_position = (t + first_audio()) / frc.speed_up;
+}
return _audio_content;
}
+ void seek (ContentTime time, bool accurate);
+
protected:
- void audio (boost::shared_ptr<const AudioBuffers>, ContentTime);
+ virtual ContentTime first_audio () const = 0;
+ void audio (boost::shared_ptr<const AudioBuffers>);
void flush ();
boost::shared_ptr<const AudioContent> _audio_content;
boost::shared_ptr<Resampler> _resampler;
- /* End time of last audio that we wrote to _pending; only used for flushing the resampler */
- ContentTime _last_audio;
+ AudioFrame _audio_position;
};
#endif
#include "types.h"
#include "rect.h"
+#include "util.h"
class Image;
{
public:
Decoded ()
- : content_time (0)
- , dcp_time (0)
- {}
-
- Decoded (DCPTime ct)
- : content_time (ct)
- , dcp_time (0)
+ : dcp_time (0)
{}
virtual ~Decoded () {}
- virtual void set_dcp_times (float speed_up, DCPTime offset) = 0;
+ virtual void set_dcp_times (VideoFrame, AudioFrame, FrameRateChange, DCPTime) = 0;
- ContentTime content_time;
DCPTime dcp_time;
};
DecodedVideo ()
: eyes (EYES_BOTH)
, same (false)
+ , frame (0)
{}
- DecodedVideo (boost::shared_ptr<const Image> im, Eyes e, bool s, ContentTime ct)
- : Decoded (ct)
- , image (im)
+ DecodedVideo (boost::shared_ptr<const Image> im, Eyes e, bool s, VideoFrame f)
+ : image (im)
, eyes (e)
, same (s)
+ , frame (f)
{}
- void set_dcp_times (float speed_up, DCPTime offset) {
- dcp_time = rint (content_time / speed_up) + offset;
+ void set_dcp_times (VideoFrame video_frame_rate, AudioFrame, FrameRateChange frc, DCPTime offset)
+ {
+ dcp_time = frame * TIME_HZ * frc.factor() / video_frame_rate + offset;
}
boost::shared_ptr<const Image> image;
Eyes eyes;
bool same;
+ VideoFrame frame;
};
class DecodedAudio : public Decoded
{
public:
- DecodedAudio (boost::shared_ptr<const AudioBuffers> d, ContentTime ct)
- : Decoded (ct)
- , data (d)
+ DecodedAudio (boost::shared_ptr<const AudioBuffers> d, AudioFrame f)
+ : data (d)
+ , frame (f)
{}
- void set_dcp_times (float speed_up, DCPTime offset) {
- dcp_time = rint (content_time / speed_up) + offset;
+ void set_dcp_times (VideoFrame, AudioFrame audio_frame_rate, FrameRateChange, DCPTime offset)
+ {
+ dcp_time = frame * TIME_HZ / audio_frame_rate + offset;
}
boost::shared_ptr<const AudioBuffers> data;
+ AudioFrame frame;
};
class DecodedSubtitle : public Decoded
{
public:
DecodedSubtitle ()
- : content_time_to (0)
+ : content_time (0)
+ , content_time_to (0)
, dcp_time_to (0)
{}
DecodedSubtitle (boost::shared_ptr<Image> im, dcpomatic::Rect<double> r, ContentTime f, ContentTime t)
- : Decoded (f)
- , image (im)
+ : image (im)
, rect (r)
+ , content_time (f)
, content_time_to (t)
, dcp_time_to (0)
{}
- void set_dcp_times (float speed_up, DCPTime offset) {
- dcp_time = rint (content_time / speed_up) + offset;
- dcp_time_to = rint (content_time_to / speed_up) + offset;
+ void set_dcp_times (VideoFrame, AudioFrame, FrameRateChange frc, DCPTime offset)
+ {
+ dcp_time = rint (content_time / frc.speed_up) + offset;
+ dcp_time_to = rint (content_time_to / frc.speed_up) + offset;
}
boost::shared_ptr<Image> image;
dcpomatic::Rect<double> rect;
+ ContentTime content_time;
ContentTime content_time_to;
DCPTime dcp_time_to;
};
_done = pass ();
}
- if (_done) {
+ if (_done && _pending.empty ()) {
return shared_ptr<Decoded> ();
}
, _subtitle_codec (0)
, _decode_video (video)
, _decode_audio (audio)
- , _video_pts_offset (0)
- , _audio_pts_offset (0)
+ , _pts_offset (0)
{
setup_subtitle ();
Then we remove big initial gaps in PTS and we allow our
insertion of black frames to work.
- We will do:
- audio_pts_to_use = audio_pts_from_ffmpeg + audio_pts_offset;
- video_pts_to_use = video_pts_from_ffmpeg + video_pts_offset;
+ We will do pts_to_use = pts_from_ffmpeg + pts_offset;
*/
bool const have_video = video && c->first_video();
/* First, make one of them start at 0 */
if (have_audio && have_video) {
- _video_pts_offset = _audio_pts_offset = - min (c->first_video().get(), c->audio_stream()->first_audio.get());
+ _pts_offset = - min (c->first_video().get(), c->audio_stream()->first_audio.get());
} else if (have_video) {
- _video_pts_offset = - c->first_video().get();
+ _pts_offset = - c->first_video().get();
} else if (have_audio) {
- _audio_pts_offset = - c->audio_stream()->first_audio.get();
+ _pts_offset = - c->audio_stream()->first_audio.get();
}
/* Now adjust both so that the video pts starts on a frame */
if (have_video && have_audio) {
- double first_video = c->first_video().get() + _video_pts_offset;
+ double first_video = c->first_video().get() + _pts_offset;
double const old_first_video = first_video;
/* Round the first video up to a frame boundary */
first_video = ceil (first_video * c->video_frame_rate()) / c->video_frame_rate ();
}
- _video_pts_offset += first_video - old_first_video;
- _audio_pts_offset += first_video - old_first_video;
+ _pts_offset += first_video - old_first_video;
}
}
r = avcodec_decode_video2 (video_codec_context(), _frame, &finished, &_packet);
if (r >= 0 && finished) {
last_video = rint (
- (av_frame_get_best_effort_timestamp (_frame) * time_base + _video_pts_offset) * TIME_HZ
+ (av_frame_get_best_effort_timestamp (_frame) * time_base + _pts_offset) * TIME_HZ
);
}
-
+
} else if (_ffmpeg_content->audio_stream() && _packet.stream_index == _ffmpeg_content->audio_stream()->index (_format_context)) {
AVPacket copy_packet = _packet;
while (copy_packet.size > 0) {
r = avcodec_decode_audio4 (audio_codec_context(), _frame, &finished, &_packet);
if (r >= 0 && finished) {
last_audio = rint (
- (av_frame_get_best_effort_timestamp (_frame) * time_base + _audio_pts_offset) * TIME_HZ
+ (av_frame_get_best_effort_timestamp (_frame) * time_base + _pts_offset) * TIME_HZ
);
}
void
FFmpegDecoder::seek_and_flush (ContentTime t)
{
- int64_t s = ((double (t) / TIME_HZ) - _video_pts_offset) /
+ int64_t s = ((double (t) / TIME_HZ) - _pts_offset) /
av_q2d (_format_context->streams[_video_stream]->time_base);
-
+
if (_ffmpeg_content->audio_stream ()) {
s = min (
s, int64_t (
- ((double (t) / TIME_HZ) - _audio_pts_offset) /
+ ((double (t) / TIME_HZ) - _pts_offset) /
av_q2d (_ffmpeg_content->audio_stream()->stream(_format_context)->time_base)
)
);
}
+ /* Ridiculous empirical hack */
+ s--;
+
av_seek_frame (_format_context, _video_stream, s, AVSEEK_FLAG_BACKWARD);
avcodec_flush_buffers (video_codec_context());
FFmpegDecoder::seek (ContentTime time, bool accurate)
{
Decoder::seek (time, accurate);
+ AudioDecoder::seek (time, accurate);
/* If we are doing an accurate seek, our initial shot will be 200ms (200 being
a number plucked from the air) earlier than we want to end up. The loop below
if (initial_seek < 0) {
initial_seek = 0;
}
-
+
/* Initial seek time in the video stream's timebase */
seek_and_flush (initial_seek);
}
int const N = minimal_run (boost::bind (&FFmpegDecoder::seek_overrun_finished, this, time, _1, _2));
-
+
seek_and_flush (initial_seek);
if (N > 0) {
minimal_run (boost::bind (&FFmpegDecoder::seek_final_finished, this, N - 1, _3));
}
if (frame_finished) {
- ContentTime const t = rint (
- (av_q2d (_format_context->streams[copy_packet.stream_index]->time_base)
- * av_frame_get_best_effort_timestamp(_frame) + _audio_pts_offset) * TIME_HZ
- );
-
int const data_size = av_samples_get_buffer_size (
0, audio_codec_context()->channels, _frame->nb_samples, audio_sample_format (), 1
);
- audio (deinterleave_audio (_frame->data, data_size), t);
+ audio (deinterleave_audio (_frame->data, data_size));
}
copy_packet.data += decode_result;
}
if (i->second != AV_NOPTS_VALUE) {
- ContentTime const t = rint ((i->second * av_q2d (_format_context->streams[_video_stream]->time_base) + _video_pts_offset) * TIME_HZ);
- video (image, false, t);
+ double const pts = i->second * av_q2d (_format_context->streams[_video_stream]->time_base) + _pts_offset;
+ VideoFrame const f = rint (pts * _ffmpeg_content->video_frame_rate ());
+ video (image, false, f);
} else {
shared_ptr<const Film> film = _film.lock ();
assert (film);
avsubtitle_free (&sub);
}
+
+ContentTime
+FFmpegDecoder::first_audio () const
+{
+ if (!_ffmpeg_content->audio_stream ()) {
+ return 0;
+ }
+
+ return _ffmpeg_content->audio_stream()->first_audio.get_value_or(0) + _pts_offset;
+}
bool pass ();
void flush ();
+ ContentTime first_audio () const;
void setup_subtitle ();
bool _decode_video;
bool _decode_audio;
- double _video_pts_offset;
- double _audio_pts_offset;
+ double _pts_offset;
};
}
if (_image && _image_content->still ()) {
- video (_image, true, _video_position * TIME_HZ / _video_content->video_frame_rate ());
+ video (_image, true, _video_position);
++_video_position;
return false;
}
delete magick_image;
- video (_image, false, _video_position * TIME_HZ / _video_content->video_frame_rate ());
+ video (_image, false, _video_position);
++_video_position;
return false;
boost::shared_ptr<const ImageContent> _image_content;
boost::shared_ptr<Image> _image;
- ContentTime _video_position;
+ VideoFrame _video_position;
};
break;
}
- dec->set_dcp_times ((*i)->frc.speed_up, offset);
+ dec->set_dcp_times (_film->video_frame_rate(), _film->audio_frame_rate(), (*i)->frc, offset);
DCPTime const t = dec->dcp_time - offset;
if (t >= (*i)->content->full_length() - (*i)->content->trim_end ()) {
- /* In the end-trimmed part; decoder has nothing eles to give us */
+ /* In the end-trimmed part; decoder has nothing else to give us */
dec.reset ();
done = true;
} else if (t >= (*i)->content->trim_start ()) {
throw DecodeError (_("could not open audio file for reading"));
}
- _done = 0;
_remaining = _info.frames;
}
}
data->set_frames (this_time);
- audio (data, _done * TIME_HZ / audio_frame_rate ());
- _done += this_time;
+ audio (data);
_remaining -= this_time;
return _remaining == 0;
SndfileDecoder::seek (ContentTime t, bool accurate)
{
Decoder::seek (t, accurate);
+ AudioDecoder::seek (t, accurate);
- _done = t * audio_frame_rate() / TIME_HZ;
_remaining = _info.frames - _done;
}
int audio_frame_rate () const;
private:
- bool has_audio () const {
- return true;
+ ContentTime first_audio () const {
+ return 0;
}
+
bool pass ();
boost::shared_ptr<const SndfileContent> _sndfile_content;
SNDFILE* _sndfile;
SF_INFO _info;
- AudioFrame _done;
AudioFrame _remaining;
float* _deinterleave_buffer;
};
/** Called by subclasses when they have a video frame ready */
void
-VideoDecoder::video (shared_ptr<const Image> image, bool same, ContentTime time)
+VideoDecoder::video (shared_ptr<const Image> image, bool same, VideoFrame frame)
{
switch (_video_content->video_frame_type ()) {
case VIDEO_FRAME_TYPE_2D:
- _pending.push_back (shared_ptr<DecodedVideo> (new DecodedVideo (image, EYES_BOTH, same, time)));
+ _pending.push_back (shared_ptr<DecodedVideo> (new DecodedVideo (image, EYES_BOTH, same, frame)));
break;
case VIDEO_FRAME_TYPE_3D_LEFT_RIGHT:
{
int const half = image->size().width / 2;
- _pending.push_back (shared_ptr<DecodedVideo> (new DecodedVideo (image->crop (Crop (0, half, 0, 0), true), EYES_LEFT, same, time)));
- _pending.push_back (shared_ptr<DecodedVideo> (new DecodedVideo (image->crop (Crop (half, 0, 0, 0), true), EYES_RIGHT, same, time)));
+ _pending.push_back (shared_ptr<DecodedVideo> (new DecodedVideo (image->crop (Crop (0, half, 0, 0), true), EYES_LEFT, same, frame)));
+ _pending.push_back (shared_ptr<DecodedVideo> (new DecodedVideo (image->crop (Crop (half, 0, 0, 0), true), EYES_RIGHT, same, frame)));
break;
}
case VIDEO_FRAME_TYPE_3D_TOP_BOTTOM:
{
int const half = image->size().height / 2;
- _pending.push_back (shared_ptr<DecodedVideo> (new DecodedVideo (image->crop (Crop (0, 0, 0, half), true), EYES_LEFT, same, time)));
- _pending.push_back (shared_ptr<DecodedVideo> (new DecodedVideo (image->crop (Crop (0, 0, half, 0), true), EYES_RIGHT, same, time)));
+ _pending.push_back (shared_ptr<DecodedVideo> (new DecodedVideo (image->crop (Crop (0, 0, 0, half), true), EYES_LEFT, same, frame)));
+ _pending.push_back (shared_ptr<DecodedVideo> (new DecodedVideo (image->crop (Crop (0, 0, half, 0), true), EYES_RIGHT, same, frame)));
break;
}
+ default:
+ assert (false);
}
}
protected:
- void video (boost::shared_ptr<const Image>, bool, ContentTime);
+ void video (boost::shared_ptr<const Image>, bool, VideoFrame);
boost::shared_ptr<const VideoContent> _video_content;
};
content->_first_video = 0;
content->_audio_stream->first_audio = 0;
FFmpegDecoder decoder (film, content, true, true);
- BOOST_CHECK_EQUAL (decoder._video_pts_offset, 0);
- BOOST_CHECK_EQUAL (decoder._audio_pts_offset, 0);
+ BOOST_CHECK_EQUAL (decoder._pts_offset, 0);
}
{
content->_first_video = 600;
content->_audio_stream->first_audio = 600;
FFmpegDecoder decoder (film, content, true, true);
- BOOST_CHECK_EQUAL (decoder._video_pts_offset, -600);
- BOOST_CHECK_EQUAL (decoder._audio_pts_offset, -600);
+ BOOST_CHECK_EQUAL (decoder._pts_offset, -600);
}
{
content->_first_video = 1.0 / 24.0;
content->_audio_stream->first_audio = 0;
FFmpegDecoder decoder (film, content, true, true);
- BOOST_CHECK_EQUAL (decoder._video_pts_offset, 0);
- BOOST_CHECK_EQUAL (decoder._audio_pts_offset, 0);
+ BOOST_CHECK_EQUAL (decoder._pts_offset, 0);
}
{
content->_first_video = frame + 0.0215;
content->_audio_stream->first_audio = 0;
FFmpegDecoder decoder (film, content, true, true);
- BOOST_CHECK_CLOSE (decoder._video_pts_offset, (frame - 0.0215), 0.00001);
- BOOST_CHECK_CLOSE (decoder._audio_pts_offset, (frame - 0.0215), 0.00001);
+ BOOST_CHECK_CLOSE (decoder._pts_offset, (frame - 0.0215), 0.00001);
}
{
content->_first_video = frame + 0.0215 + 4.1;
content->_audio_stream->first_audio = 4.1;
FFmpegDecoder decoder (film, content, true, true);
- BOOST_CHECK_EQUAL (decoder._video_pts_offset, (frame - 0.0215) - 4.1);
- BOOST_CHECK_EQUAL (decoder._audio_pts_offset, (frame - 0.0215) - 4.1);
+ BOOST_CHECK_EQUAL (decoder._pts_offset, (frame - 0.0215) - 4.1);
}
}
using std::cout;
using boost::shared_ptr;
+using boost::dynamic_pointer_cast;
BOOST_AUTO_TEST_CASE (seek_zero_test)
{
wait_for_jobs ();
FFmpegDecoder decoder (film, content, true, false);
- shared_ptr<Decoded> a = decoder.peek ();
- cout << a->content_time << "\n";
+ shared_ptr<DecodedVideo> a = dynamic_pointer_cast<DecodedVideo> (decoder.peek ());
decoder.seek (0, true);
- shared_ptr<Decoded> b = decoder.peek ();
- cout << b->content_time << "\n";
+ shared_ptr<DecodedVideo> b = dynamic_pointer_cast<DecodedVideo> (decoder.peek ());
/* a will be after no seek, and b after a seek to zero, which should
have the same effect.
*/
- BOOST_CHECK_EQUAL (a->content_time, b->content_time);
+ BOOST_CHECK_EQUAL (a->frame, b->frame);
}