From 54d17e98a597334bf1ba2615e3eb6191088f606f Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Mon, 15 Feb 2016 08:40:14 +0000 Subject: [PATCH] Support SSA subtitles embedded within FFmpeg files. --- ChangeLog | 4 ++ cscript | 2 +- src/lib/ffmpeg.cc | 26 +++++++++ src/lib/ffmpeg.h | 1 + src/lib/ffmpeg_content.cc | 33 ++++++++++-- src/lib/ffmpeg_content.h | 3 +- src/lib/ffmpeg_decoder.cc | 80 +++++++++++++++++++++++++-- src/lib/ffmpeg_decoder.h | 1 + src/lib/ffmpeg_examiner.cc | 42 +++++++++++---- src/lib/ffmpeg_examiner.h | 5 +- src/lib/ffmpeg_subtitle_stream.cc | 89 +++++++++++++++++++++++++------ src/lib/ffmpeg_subtitle_stream.h | 22 ++++++-- src/lib/player.cc | 1 + src/lib/render_subtitles.cc | 2 +- 14 files changed, 269 insertions(+), 42 deletions(-) diff --git a/ChangeLog b/ChangeLog index a96740d1f..63a187da4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,6 +3,10 @@ * Fix exception when analysing audio of projects with more than 8 DCP channels. +2016-02-15 c.hetherington + + * Support SSA subtitles embedded within FFmpeg files. + 2016-02-12 Carl Hetherington * Add basic support for SSA (SubStation Alpha) subtitles (#128). diff --git a/cscript b/cscript index b9b683f04..81e617800 100644 --- a/cscript +++ b/cscript @@ -281,7 +281,7 @@ def dependencies(target): return (('ffmpeg-cdist', '96d67de', ffmpeg_options), ('libdcp', '4e38f82'), - ('libsub', 'd79b29f')) + ('libsub', '9cefa0f')) def configure_options(target): opt = '' diff --git a/src/lib/ffmpeg.cc b/src/lib/ffmpeg.cc index 954aff728..29dda1b9e 100644 --- a/src/lib/ffmpeg.cc +++ b/src/lib/ffmpeg.cc @@ -288,6 +288,32 @@ FFmpeg::subtitle_id (AVSubtitle const & sub) return digester.get (); } +bool +FFmpeg::subtitle_is_image (AVSubtitle const & sub) +{ + bool image = false; + bool text = false; + + for (unsigned int i = 0; i < sub.num_rects; ++i) { + switch (sub.rects[i]->type) { + case SUBTITLE_BITMAP: + image = true; + break; + case SUBTITLE_TEXT: + case SUBTITLE_ASS: + text = true; + break; + default: + break; + } + } + + /* We can't cope with mixed image/text in one AVSubtitle */ + DCPOMATIC_ASSERT (!image || !text); + + return image; +} + /** Compute the pts offset to use given a set of audio streams and some video details. * Sometimes these parameters will have just been determined by an Examiner, sometimes * they will have been retrieved from a piece of Content, hence the need for this method diff --git a/src/lib/ffmpeg.h b/src/lib/ffmpeg.h index 0b195268a..9795b2229 100644 --- a/src/lib/ffmpeg.h +++ b/src/lib/ffmpeg.h @@ -58,6 +58,7 @@ protected: static FFmpegSubtitlePeriod subtitle_period (AVSubtitle const & sub); static std::string subtitle_id (AVSubtitle const & sub); + static bool subtitle_is_image (AVSubtitle const & sub); boost::shared_ptr _ffmpeg_content; diff --git a/src/lib/ffmpeg_content.cc b/src/lib/ffmpeg_content.cc index b47116bdc..5cd5d3729 100644 --- a/src/lib/ffmpeg_content.cc +++ b/src/lib/ffmpeg_content.cc @@ -322,26 +322,49 @@ FFmpegContent::identifier () const } list -FFmpegContent::subtitles_during (ContentTimePeriod period, bool starting) const +FFmpegContent::image_subtitles_during (ContentTimePeriod period, bool starting) const { shared_ptr stream = subtitle_stream (); if (!stream) { return list (); } - return stream->subtitles_during (period, starting); + return stream->image_subtitles_during (period, starting); +} + +list +FFmpegContent::text_subtitles_during (ContentTimePeriod period, bool starting) const +{ + shared_ptr stream = subtitle_stream (); + if (!stream) { + return list (); + } + + return stream->text_subtitles_during (period, starting); } bool -FFmpegContent::has_text_subtitles () const +FFmpegContent::has_image_subtitles () const { + BOOST_FOREACH (shared_ptr i, subtitle_streams()) { + if (i->has_image_subtitles()) { + return true; + } + } + return false; } bool -FFmpegContent::has_image_subtitles () const +FFmpegContent::has_text_subtitles () const { - return !subtitle_streams().empty (); + BOOST_FOREACH (shared_ptr i, subtitle_streams()) { + if (i->has_text_subtitles()) { + return true; + } + } + + return false; } void diff --git a/src/lib/ffmpeg_content.h b/src/lib/ffmpeg_content.h index 5437c5bf3..8bd84c144 100644 --- a/src/lib/ffmpeg_content.h +++ b/src/lib/ffmpeg_content.h @@ -102,7 +102,8 @@ public: return _first_video; } - std::list subtitles_during (ContentTimePeriod, bool starting) const; + std::list image_subtitles_during (ContentTimePeriod, bool starting) const; + std::list text_subtitles_during (ContentTimePeriod, bool starting) const; protected: void add_properties (std::list > &) const; diff --git a/src/lib/ffmpeg_decoder.cc b/src/lib/ffmpeg_decoder.cc index f97392ef7..a1d90b2ba 100644 --- a/src/lib/ffmpeg_decoder.cc +++ b/src/lib/ffmpeg_decoder.cc @@ -36,11 +36,16 @@ #include "film.h" #include "md5_digester.h" #include "compose.hpp" +#include +#include +#include +#include extern "C" { #include #include } #include +#include #include #include #include @@ -54,12 +59,15 @@ extern "C" { #define LOG_WARNING(...) _log->log (String::compose (__VA_ARGS__), LogEntry::TYPE_WARNING); using std::cout; +using std::string; using std::vector; using std::list; using std::min; using std::pair; using std::max; using boost::shared_ptr; +using boost::is_any_of; +using boost::split; using dcp::Size; FFmpegDecoder::FFmpegDecoder (shared_ptr c, shared_ptr log, bool fast) @@ -447,7 +455,7 @@ FFmpegDecoder::decode_subtitle_packet () cout << "XXX: SUBTITLE_TEXT " << rect->text << "\n"; break; case SUBTITLE_ASS: - cout << "XXX: SUBTITLE_ASS " << rect->ass << "\n"; + decode_ass_subtitle (rect->ass, period); break; } } @@ -458,13 +466,13 @@ FFmpegDecoder::decode_subtitle_packet () list FFmpegDecoder::image_subtitles_during (ContentTimePeriod p, bool starting) const { - return _ffmpeg_content->subtitles_during (p, starting); + return _ffmpeg_content->image_subtitles_during (p, starting); } list -FFmpegDecoder::text_subtitles_during (ContentTimePeriod, bool) const +FFmpegDecoder::text_subtitles_during (ContentTimePeriod p, bool starting) const { - return list (); + return _ffmpeg_content->text_subtitles_during (p, starting); } void @@ -505,3 +513,67 @@ FFmpegDecoder::decode_bitmap_subtitle (AVSubtitleRect const * rect, ContentTimeP image_subtitle (period, image, scaled_rect); } + +void +FFmpegDecoder::decode_ass_subtitle (string ass, ContentTimePeriod period) +{ + /* We have no styles and no Format: line, so I'm assuming that FFmpeg + produces a single format of Dialogue: lines... + */ + + vector bits; + split (bits, ass, is_any_of (",")); + if (bits.size() < 10) { + return; + } + + sub::RawSubtitle base; + list raw = sub::SSAReader::parse_line (base, bits[9]); + list subs = sub::collect > (raw); + + /* XXX: lots of this is copied from TextSubtitle; there should probably be some sharing */ + + /* Highest line index in this subtitle */ + int highest = 0; + BOOST_FOREACH (sub::Subtitle i, subs) { + BOOST_FOREACH (sub::Line j, i.lines) { + DCPOMATIC_ASSERT (j.vertical_position.reference && j.vertical_position.reference.get() == sub::TOP_OF_SUBTITLE); + DCPOMATIC_ASSERT (j.vertical_position.line); + highest = max (highest, j.vertical_position.line.get()); + } + } + + list ss; + + BOOST_FOREACH (sub::Subtitle i, sub::collect > (sub::SSAReader::parse_line (base, bits[9]))) { + BOOST_FOREACH (sub::Line j, i.lines) { + BOOST_FOREACH (sub::Block k, j.blocks) { + ss.push_back ( + dcp::SubtitleString ( + boost::optional (), + k.italic, + dcp::Colour (255, 255, 255), + 60, + 1, + dcp::Time (i.from.seconds(), 1000), + dcp::Time (i.to.seconds(), 1000), + 0, + dcp::HALIGN_CENTER, + /* This 1.015 is an arbitrary value to lift the bottom sub off the bottom + of the screen a bit to a pleasing degree. + */ + 1.015 - ((1 + highest - j.vertical_position.line.get()) * 1.5 / 22), + dcp::VALIGN_TOP, + k.text, + static_cast (0), + dcp::Colour (255, 255, 255), + dcp::Time (), + dcp::Time () + ) + ); + } + } + } + + text_subtitle (period, ss); +} diff --git a/src/lib/ffmpeg_decoder.h b/src/lib/ffmpeg_decoder.h index 4bb09cf9d..990d643a7 100644 --- a/src/lib/ffmpeg_decoder.h +++ b/src/lib/ffmpeg_decoder.h @@ -61,6 +61,7 @@ private: void decode_subtitle_packet (); void decode_bitmap_subtitle (AVSubtitleRect const * rect, ContentTimePeriod period); + void decode_ass_subtitle (std::string ass, ContentTimePeriod period); void maybe_add_subtitle (); boost::shared_ptr deinterleave_audio (boost::shared_ptr stream) const; diff --git a/src/lib/ffmpeg_examiner.cc b/src/lib/ffmpeg_examiner.cc index 48738b917..dacc652bb 100644 --- a/src/lib/ffmpeg_examiner.cc +++ b/src/lib/ffmpeg_examiner.cc @@ -145,13 +145,23 @@ FFmpegExaminer::FFmpegExaminer (shared_ptr c, shared_ptrsecond) { - i->first->add_subtitle ( - i->second->id, - ContentTimePeriod ( - i->second->time, - ContentTime::from_frames (video_length(), video_frame_rate().get_value_or (24)) - ) - ); + if (i->second->image) { + i->first->add_image_subtitle ( + i->second->id, + ContentTimePeriod ( + i->second->time, + ContentTime::from_frames (video_length(), video_frame_rate().get_value_or (24)) + ) + ); + } else { + i->first->add_text_subtitle ( + i->second->id, + ContentTimePeriod ( + i->second->time, + ContentTime::from_frames (video_length(), video_frame_rate().get_value_or (24)) + ) + ); + } } } @@ -207,23 +217,33 @@ FFmpegExaminer::subtitle_packet (AVCodecContext* context, shared_ptr= 0 && frame_finished) { string id = subtitle_id (sub); FFmpegSubtitlePeriod const period = subtitle_period (sub); + bool const image = subtitle_is_image (sub); + LastSubtitleMap::iterator last = _last_subtitle_start.find (stream); if (last != _last_subtitle_start.end() && last->second) { /* We have seen the start of a subtitle but not yet the end. Whatever this is finishes the previous subtitle, so add it */ - stream->add_subtitle (last->second->id, ContentTimePeriod (last->second->time, period.from)); + if (image) { + stream->add_image_subtitle (last->second->id, ContentTimePeriod (last->second->time, period.from)); + } else { + stream->add_text_subtitle (last->second->id, ContentTimePeriod (last->second->time, period.from)); + } if (sub.num_rects == 0) { /* This is a `proper' end-of-subtitle */ _last_subtitle_start[stream] = optional (); } else { /* This is just another subtitle, so we start again */ - _last_subtitle_start[stream] = SubtitleStart (id, period.from); + _last_subtitle_start[stream] = SubtitleStart (id, image, period.from); } } else if (sub.num_rects == 1) { if (period.to) { - stream->add_subtitle (id, ContentTimePeriod (period.from, period.to.get ())); + if (image) { + stream->add_image_subtitle (id, ContentTimePeriod (period.from, period.to.get ())); + } else { + stream->add_text_subtitle (id, ContentTimePeriod (period.from, period.to.get ())); + } } else { - _last_subtitle_start[stream] = SubtitleStart (id, period.from); + _last_subtitle_start[stream] = SubtitleStart (id, image, period.from); } } avsubtitle_free (&sub); diff --git a/src/lib/ffmpeg_examiner.h b/src/lib/ffmpeg_examiner.h index e87e11d1c..a2a80b254 100644 --- a/src/lib/ffmpeg_examiner.h +++ b/src/lib/ffmpeg_examiner.h @@ -88,12 +88,15 @@ private: struct SubtitleStart { - SubtitleStart (std::string id_, ContentTime time_) + SubtitleStart (std::string id_, bool image_, ContentTime time_) : id (id_) + , image (image_) , time (time_) {} std::string id; + /** true if it's an image subtitle, false for text */ + bool image; ContentTime time; }; diff --git a/src/lib/ffmpeg_subtitle_stream.cc b/src/lib/ffmpeg_subtitle_stream.cc index 466032b37..33759d86e 100644 --- a/src/lib/ffmpeg_subtitle_stream.cc +++ b/src/lib/ffmpeg_subtitle_stream.cc @@ -37,10 +37,10 @@ FFmpegSubtitleStream::FFmpegSubtitleStream (cxml::ConstNodePtr node, int version { if (version == 32) { BOOST_FOREACH (cxml::NodePtr i, node->node_children ("Period")) { - /* In version 32 we assumed that from times were unique, so they weer - used as identifiers. + /* In version 32 we assumed that from times were unique, so they were + used as identifiers. All subtitles were image subtitles. */ - add_subtitle ( + add_image_subtitle ( raw_convert (i->string_child ("From")), ContentTimePeriod ( ContentTime (i->number_child ("From")), @@ -49,9 +49,32 @@ FFmpegSubtitleStream::FFmpegSubtitleStream (cxml::ConstNodePtr node, int version ); } } else { - /* In version 33 we use a hash of various parts of the subtitle as the id */ + /* In version 33 we use a hash of various parts of the subtitle as the id. + was initially used for image subtitles; later we have + and + */ BOOST_FOREACH (cxml::NodePtr i, node->node_children ("Subtitle")) { - add_subtitle ( + add_image_subtitle ( + raw_convert (i->string_child ("Id")), + ContentTimePeriod ( + ContentTime (i->number_child ("From")), + ContentTime (i->number_child ("To")) + ) + ); + } + + BOOST_FOREACH (cxml::NodePtr i, node->node_children ("ImageSubtitle")) { + add_image_subtitle ( + raw_convert (i->string_child ("Id")), + ContentTimePeriod ( + ContentTime (i->number_child ("From")), + ContentTime (i->number_child ("To")) + ) + ); + } + + BOOST_FOREACH (cxml::NodePtr i, node->node_children ("TextSubtitle")) { + add_text_subtitle ( raw_convert (i->string_child ("Id")), ContentTimePeriod ( ContentTime (i->number_child ("From")), @@ -67,8 +90,15 @@ FFmpegSubtitleStream::as_xml (xmlpp::Node* root) const { FFmpegStream::as_xml (root); - for (map::const_iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) { - xmlpp::Node* node = root->add_child ("Subtitle"); + as_xml (root, _image_subtitles, "ImageSubtitle"); + as_xml (root, _text_subtitles, "TextSubtitle"); +} + +void +FFmpegSubtitleStream::as_xml (xmlpp::Node* root, PeriodMap const & subs, string node_name) const +{ + for (PeriodMap::const_iterator i = subs.begin(); i != subs.end(); ++i) { + xmlpp::Node* node = root->add_child (node_name); node->add_child("Id")->add_child_text (i->first); node->add_child("From")->add_child_text (raw_convert (i->second.from.get ())); node->add_child("To")->add_child_text (raw_convert (i->second.to.get ())); @@ -76,19 +106,38 @@ FFmpegSubtitleStream::as_xml (xmlpp::Node* root) const } void -FFmpegSubtitleStream::add_subtitle (string id, ContentTimePeriod period) +FFmpegSubtitleStream::add_image_subtitle (string id, ContentTimePeriod period) +{ + DCPOMATIC_ASSERT (_image_subtitles.find (id) == _image_subtitles.end ()); + _image_subtitles[id] = period; +} + +void +FFmpegSubtitleStream::add_text_subtitle (string id, ContentTimePeriod period) { - DCPOMATIC_ASSERT (_subtitles.find (id) == _subtitles.end ()); - _subtitles[id] = period; + DCPOMATIC_ASSERT (_text_subtitles.find (id) == _text_subtitles.end ()); + _text_subtitles[id] = period; } list -FFmpegSubtitleStream::subtitles_during (ContentTimePeriod period, bool starting) const +FFmpegSubtitleStream::image_subtitles_during (ContentTimePeriod period, bool starting) const +{ + return subtitles_during (period, starting, _image_subtitles); +} + +list +FFmpegSubtitleStream::text_subtitles_during (ContentTimePeriod period, bool starting) const +{ + return subtitles_during (period, starting, _text_subtitles); +} + +list +FFmpegSubtitleStream::subtitles_during (ContentTimePeriod period, bool starting, PeriodMap const & subs) const { list d; /* XXX: inefficient */ - for (map::const_iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) { + for (map::const_iterator i = subs.begin(); i != subs.end(); ++i) { if ((starting && period.contains (i->second.from)) || (!starting && period.overlaps (i->second))) { d.push_back (i->second); } @@ -100,8 +149,13 @@ FFmpegSubtitleStream::subtitles_during (ContentTimePeriod period, bool starting) ContentTime FFmpegSubtitleStream::find_subtitle_to (string id) const { - map::const_iterator i = _subtitles.find (id); - DCPOMATIC_ASSERT (i != _subtitles.end ()); + PeriodMap::const_iterator i = _image_subtitles.find (id); + if (i != _image_subtitles.end ()) { + return i->second.to; + } + + i = _text_subtitles.find (id); + DCPOMATIC_ASSERT (i != _text_subtitles.end ()); return i->second.to; } @@ -109,7 +163,12 @@ FFmpegSubtitleStream::find_subtitle_to (string id) const void FFmpegSubtitleStream::add_offset (ContentTime offset) { - for (map::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) { + for (PeriodMap::iterator i = _image_subtitles.begin(); i != _image_subtitles.end(); ++i) { + i->second.from += offset; + i->second.to += offset; + } + + for (PeriodMap::iterator i = _text_subtitles.begin(); i != _text_subtitles.end(); ++i) { i->second.from += offset; i->second.to += offset; } diff --git a/src/lib/ffmpeg_subtitle_stream.h b/src/lib/ffmpeg_subtitle_stream.h index 6cd5318d0..688aaa993 100644 --- a/src/lib/ffmpeg_subtitle_stream.h +++ b/src/lib/ffmpeg_subtitle_stream.h @@ -31,11 +31,27 @@ public: void as_xml (xmlpp::Node *) const; - void add_subtitle (std::string id, ContentTimePeriod period); - std::list subtitles_during (ContentTimePeriod period, bool starting) const; + void add_image_subtitle (std::string id, ContentTimePeriod period); + void add_text_subtitle (std::string id, ContentTimePeriod period); + std::list image_subtitles_during (ContentTimePeriod period, bool starting) const; + std::list text_subtitles_during (ContentTimePeriod period, bool starting) const; ContentTime find_subtitle_to (std::string id) const; void add_offset (ContentTime offset); + bool has_image_subtitles () const { + return !_image_subtitles.empty (); + } + bool has_text_subtitles () const { + return !_text_subtitles.empty (); + } + private: - std::map _subtitles; + + typedef std::map PeriodMap; + + void as_xml (xmlpp::Node *, PeriodMap const & subs, std::string node) const; + std::list subtitles_during (ContentTimePeriod period, bool starting, PeriodMap const & subs) const; + + PeriodMap _image_subtitles; + PeriodMap _text_subtitles; }; diff --git a/src/lib/player.cc b/src/lib/player.cc index fc1332573..2028758ac 100644 --- a/src/lib/player.cc +++ b/src/lib/player.cc @@ -373,6 +373,7 @@ Player::get_video (DCPTime time, bool accurate) if (!ps.text.empty ()) { list s = render_subtitles (ps.text, ps.fonts, _video_container_size); copy (s.begin (), s.end (), back_inserter (sub_images)); + cout << "got " << s.size() << " text subs rendered to images.\n"; } optional subtitles; diff --git a/src/lib/render_subtitles.cc b/src/lib/render_subtitles.cc index 26f41c89e..eb67d68a5 100644 --- a/src/lib/render_subtitles.cc +++ b/src/lib/render_subtitles.cc @@ -279,7 +279,7 @@ render_line (list subtitles, list > fonts, break; } - return PositionImage (image, Position (x, y)); + return PositionImage (image, Position (max (0, x), max (0, y))); } list -- 2.30.2