Use a separate AVFrame for each stream when decoding.

This seems to be what ffplay does and it feels like it makes sense
as frames may be built from multiple packets AFAICS.

diff --git a/src/lib/ffmpeg.cc b/src/lib/ffmpeg.cc
index eed9ab94c161fb90fdcb4d613092fc49d78860d6..0f63ea1727ee5374ce3d41ecd3c96a9f874386cf 100644 (file)
--- a/src/lib/ffmpeg.cc
+++ b/src/lib/ffmpeg.cc
@@ -72,7 +72,11 @@ FFmpeg::~FFmpeg ()
                avcodec_free_context (&i);
        }
 
-       av_frame_free (&_frame);
+       av_frame_free (&_video_frame);
+       for (auto& audio_frame: _audio_frame) {
+               av_frame_free (&audio_frame.second);
+       }
+
        avformat_close_input (&_format_context);
 }
 
@@ -188,8 +192,8 @@ FFmpeg::setup_general ()
                }
        }
 
-       _frame = av_frame_alloc ();
-       if (_frame == 0) {
+       _video_frame = av_frame_alloc ();
+       if (_video_frame == nullptr) {
                throw std::bad_alloc ();
        }
 }
@@ -354,3 +358,23 @@ FFmpeg::pts_offset (vector<shared_ptr<FFmpegAudioStream>> audio_streams, optiona
 
        return po;
 }
+
+
+AVFrame *
+FFmpeg::audio_frame (shared_ptr<const FFmpegAudioStream> stream)
+{
+       auto iter = _audio_frame.find(stream);
+       if (iter != _audio_frame.end()) {
+               return iter->second;
+       }
+
+       auto frame = av_frame_alloc ();
+       if (frame == nullptr) {
+               throw std::bad_alloc();
+       }
+
+       _audio_frame[stream] = frame;
+       return frame;
+
+}
+

diff --git a/src/lib/ffmpeg.h b/src/lib/ffmpeg.h
index b0769971e3e9c9f00f4812c51b3333badd6654f8..59b2dedd5bd5d05998b67e1485f33ec33d2c2a3c 100644 (file)
--- a/src/lib/ffmpeg.h
+++ b/src/lib/ffmpeg.h
@@ -75,11 +75,14 @@ protected:
 
        AVFormatContext* _format_context = nullptr;
        std::vector<AVCodecContext*> _codec_context;
-       AVFrame* _frame = nullptr;
 
+       /** AVFrame used for decoding video */
+       AVFrame* _video_frame = nullptr;
        /** Index of video stream within AVFormatContext */
        boost::optional<int> _video_stream;
 
+       AVFrame* audio_frame (std::shared_ptr<const FFmpegAudioStream> stream);
+
        /* It would appear (though not completely verified) that one must have
           a mutex around calls to avcodec_open* and avcodec_close... and here
           it is.
@@ -92,6 +95,9 @@ private:
 
        static void ffmpeg_log_callback (void* ptr, int level, const char* fmt, va_list vl);
        static std::weak_ptr<Log> _ffmpeg_log;
+
+       /** AVFrames used for decoding audio streams; accessed with audio_frame() */
+       std::map<std::shared_ptr<const FFmpegAudioStream>, AVFrame*> _audio_frame;
 };
 
 

diff --git a/src/lib/ffmpeg_decoder.cc b/src/lib/ffmpeg_decoder.cc
index ea961a894d5446b5a81731ea86909a14cc918635..e5685f6611d60665367b46b97e5f66a06634d113 100644 (file)
--- a/src/lib/ffmpeg_decoder.cc
+++ b/src/lib/ffmpeg_decoder.cc
@@ -122,7 +122,7 @@ FFmpegDecoder::flush ()
                        /* EOF can happen if we've already sent a flush packet */
                        throw DecodeError (N_("avcodec_send_packet"), N_("FFmpegDecoder::flush"), r);
                }
-               r = avcodec_receive_frame (context, _frame);
+               r = avcodec_receive_frame (context, audio_frame(i));
                if (r >= 0) {
                        process_audio_frame (i);
                        did_something = true;
@@ -447,10 +447,11 @@ FFmpegDecoder::audio_stream_from_index (int index) const
 void
 FFmpegDecoder::process_audio_frame (shared_ptr<FFmpegAudioStream> stream)
 {
-       auto data = deinterleave_audio (_frame);
+       auto frame = audio_frame (stream);
+       auto data = deinterleave_audio (frame);
 
        ContentTime ct;
-       if (_frame->pts == AV_NOPTS_VALUE) {
+       if (frame->pts == AV_NOPTS_VALUE) {
                /* In some streams we see not every frame coming through with a timestamp; for those
                   that have AV_NOPTS_VALUE we need to work out the timestamp ourselves.  This is
                   particularly noticeable with TrueHD streams (see #1111).
@@ -460,7 +461,7 @@ FFmpegDecoder::process_audio_frame (shared_ptr<FFmpegAudioStream> stream)
                }
        } else {
                ct = ContentTime::from_seconds (
-                       _frame->best_effort_timestamp *
+                       frame->best_effort_timestamp *
                        av_q2d (stream->stream(_format_context)->time_base))
                        + _pts_offset;
        }
@@ -481,7 +482,7 @@ FFmpegDecoder::process_audio_frame (shared_ptr<FFmpegAudioStream> stream)
                        to_string(ct),
                        data->frames(),
                        stream->id(),
-                       _frame->best_effort_timestamp,
+                       frame->best_effort_timestamp,
                        av_q2d(stream->stream(_format_context)->time_base),
                        to_string(_pts_offset)
                        );
@@ -503,13 +504,14 @@ FFmpegDecoder::decode_and_process_audio_packet (AVPacket* packet)
        }
 
        auto context = _codec_context[stream->index(_format_context)];
+       auto frame = audio_frame (stream);
 
        int r = avcodec_send_packet (context, packet);
        if (r < 0) {
                LOG_WARNING("avcodec_send_packet returned %1 for an audio packet", r);
        }
        while (r >= 0) {
-               r = avcodec_receive_frame (context, _frame);
+               r = avcodec_receive_frame (context, frame);
                if (r == AVERROR(EAGAIN)) {
                        /* More input is required */
                        return;
@@ -536,7 +538,7 @@ FFmpegDecoder::decode_and_process_video_packet (AVPacket* packet)
                LOG_WARNING("avcodec_send_packet returned %1 for a video packet", r);
        }
 
-       r = avcodec_receive_frame (context, _frame);
+       r = avcodec_receive_frame (context, _video_frame);
        if (r == AVERROR(EAGAIN) || r == AVERROR_EOF || (r < 0 && !packet)) {
                /* More input is required, no more frames are coming, or we are flushing and there was
                 * some error which we just want to ignore.
@@ -553,21 +555,21 @@ FFmpegDecoder::decode_and_process_video_packet (AVPacket* packet)
        shared_ptr<VideoFilterGraph> graph;
 
        auto i = _filter_graphs.begin();
-       while (i != _filter_graphs.end() && !(*i)->can_process (dcp::Size (_frame->width, _frame->height), (AVPixelFormat) _frame->format)) {
+       while (i != _filter_graphs.end() && !(*i)->can_process(dcp::Size(_video_frame->width, _video_frame->height), (AVPixelFormat) _video_frame->format)) {
                ++i;
        }
 
        if (i == _filter_graphs.end ()) {
                dcp::Fraction vfr (lrint(_ffmpeg_content->video_frame_rate().get() * 1000), 1000);
-               graph = make_shared<VideoFilterGraph>(dcp::Size (_frame->width, _frame->height), (AVPixelFormat) _frame->format, vfr);
+               graph = make_shared<VideoFilterGraph>(dcp::Size(_video_frame->width, _video_frame->height), (AVPixelFormat) _video_frame->format, vfr);
                graph->setup (_ffmpeg_content->filters ());
                _filter_graphs.push_back (graph);
-               LOG_GENERAL (N_("New graph for %1x%2, pixel format %3"), _frame->width, _frame->height, _frame->format);
+               LOG_GENERAL (N_("New graph for %1x%2, pixel format %3"), _video_frame->width, _video_frame->height, _video_frame->format);
        } else {
                graph = *i;
        }
 
-       auto images = graph->process (_frame);
+       auto images = graph->process (_video_frame);
 
        for (auto const& i: images) {
 

diff --git a/src/lib/ffmpeg_examiner.cc b/src/lib/ffmpeg_examiner.cc
index 0a236d836be46b4f5baacdee175201e154de6cab..5e53f0974cd61865d8c09287c4b806590ca65d2f 100644 (file)
--- a/src/lib/ffmpeg_examiner.cc
+++ b/src/lib/ffmpeg_examiner.cc
@@ -229,7 +229,7 @@ FFmpegExaminer::video_packet (AVCodecContext* context, string& temporal_referenc
                throw DecodeError (N_("avcodec_send_packet"), N_("FFmpegExaminer::video_packet"), r);
        }
 
-       r = avcodec_receive_frame (context, _frame);
+       r = avcodec_receive_frame (context, _video_frame);
        if (r == AVERROR(EAGAIN)) {
                /* More input is required */
                return true;
@@ -239,16 +239,17 @@ FFmpegExaminer::video_packet (AVCodecContext* context, string& temporal_referenc
        }
 
        if (!_first_video) {
-               _first_video = frame_time (_format_context->streams[_video_stream.get()]);
+               _first_video = frame_time (_video_frame, _format_context->streams[_video_stream.get()]);
        }
        if (_need_video_length) {
                _video_length = frame_time (
+                       _video_frame,
                        _format_context->streams[_video_stream.get()]
                        ).get_value_or (ContentTime ()).frames_round (video_frame_rate().get ());
        }
        if (temporal_reference.size() < (PULLDOWN_CHECK_FRAMES * 2)) {
-               temporal_reference += (_frame->top_field_first ? "T" : "B");
-               temporal_reference += (_frame->repeat_pict ? "3" : "2");
+               temporal_reference += (_video_frame->top_field_first ? "T" : "B");
+               temporal_reference += (_video_frame->repeat_pict ? "3" : "2");
        }
 
        return true;
@@ -271,22 +272,24 @@ FFmpegExaminer::audio_packet (AVCodecContext* context, shared_ptr<FFmpegAudioStr
                throw DecodeError (N_("avcodec_send_packet"), N_("FFmpegExaminer::audio_packet"), r);
        }
 
-       if (avcodec_receive_frame (context, _frame) < 0) {
+       auto frame = audio_frame (stream);
+
+       if (avcodec_receive_frame (context, frame) < 0) {
                return;
        }
 
-       stream->first_audio = frame_time (stream->stream(_format_context));
+       stream->first_audio = frame_time (frame, stream->stream(_format_context));
 }
 
 
 optional<ContentTime>
-FFmpegExaminer::frame_time (AVStream* s) const
+FFmpegExaminer::frame_time (AVFrame* frame, AVStream* stream) const
 {
        optional<ContentTime> t;
 
-       int64_t const bet = _frame->best_effort_timestamp;
+       int64_t const bet = frame->best_effort_timestamp;
        if (bet != AV_NOPTS_VALUE) {
-               t = ContentTime::from_seconds (bet * av_q2d (s->time_base));
+               t = ContentTime::from_seconds (bet * av_q2d(stream->time_base));
        }
 
        return t;

diff --git a/src/lib/ffmpeg_examiner.h b/src/lib/ffmpeg_examiner.h
index f978eb52bc00e63b55f3ab34034f57ea2893cdb8..36efc92b1b6e1090aa1db490404795cf028bca2f 100644 (file)
--- a/src/lib/ffmpeg_examiner.h
+++ b/src/lib/ffmpeg_examiner.h
@@ -91,7 +91,7 @@ private:
 
        std::string stream_name (AVStream* s) const;
        std::string subtitle_stream_name (AVStream* s) const;
-       boost::optional<dcpomatic::ContentTime> frame_time (AVStream* s) const;
+       boost::optional<dcpomatic::ContentTime> frame_time (AVFrame* frame, AVStream* stream) const;
 
        std::vector<std::shared_ptr<FFmpegSubtitleStream>> _subtitle_streams;
        std::vector<std::shared_ptr<FFmpegAudioStream>> _audio_streams;