X-Git-Url: https://main.carlh.net/gitweb/?a=blobdiff_plain;f=src%2Fverify.cc;h=97758e6192e632aae3bf4c498c167a90120818ad;hb=2c1faeb15715794525f48110c2b8a9df96b387c1;hp=9dc87eece0b553dea6004c37035293bac25beb8c;hpb=d4519ea7898ccb878a297126bbb45c7b65cccd08;p=libdcp.git diff --git a/src/verify.cc b/src/verify.cc index 9dc87eec..97758e61 100644 --- a/src/verify.cc +++ b/src/verify.cc @@ -1,5 +1,5 @@ /* - Copyright (C) 2018-2020 Carl Hetherington + Copyright (C) 2018-2021 Carl Hetherington This file is part of libdcp. @@ -31,6 +31,12 @@ files in the program, then also delete it here. */ + +/** @file src/verify.cc + * @brief dcp::verify() method and associated code + */ + + #include "verify.h" #include "dcp.h" #include "cpl.h" @@ -47,6 +53,7 @@ #include "exceptions.h" #include "compose.hpp" #include "raw_convert.h" +#include "reel_markers_asset.h" #include "smpte_subtitle_asset.h" #include #include @@ -66,28 +73,30 @@ #include #include #include -#include -#include #include #include -#include #include #include + using std::list; using std::vector; using std::string; using std::cout; using std::map; using std::max; +using std::set; using std::shared_ptr; +using std::make_shared; using boost::optional; using boost::function; using std::dynamic_pointer_cast; + using namespace dcp; using namespace xercesc; + static string xml_ch_to_string (XMLCh const * a) @@ -98,6 +107,7 @@ xml_ch_to_string (XMLCh const * a) return o; } + class XMLValidationError { public: @@ -183,7 +193,8 @@ private: list _errors; }; -class StringToXMLCh : public boost::noncopyable + +class StringToXMLCh { public: StringToXMLCh (string a) @@ -191,6 +202,9 @@ public: _buffer = XMLString::transcode(a.c_str()); } + StringToXMLCh (StringToXMLCh const&) = delete; + StringToXMLCh& operator= (StringToXMLCh const&) = delete; + ~StringToXMLCh () { XMLString::release (&_buffer); @@ -204,6 +218,7 @@ private: XMLCh* _buffer; }; + class LocalFileResolver : public EntityResolver { public: @@ -233,8 +248,8 @@ public: if (!system_id) { return 0; } - string system_id_str = xml_ch_to_string (system_id); - boost::filesystem::path p = _xsd_dtd_directory; + auto system_id_str = xml_ch_to_string (system_id); + auto p = _xsd_dtd_directory; if (_files.find(system_id_str) == _files.end()) { p /= system_id_str; } else { @@ -263,7 +278,7 @@ parse (XercesDOMParser& parser, boost::filesystem::path xml) static void -parse (XercesDOMParser& parser, std::string xml) +parse (XercesDOMParser& parser, string xml) { xercesc::MemBufInputSource buf(reinterpret_cast(xml.c_str()), xml.size(), ""); parser.parse(buf); @@ -272,7 +287,7 @@ parse (XercesDOMParser& parser, std::string xml) template void -validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, list& notes) +validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, vector& notes) { try { XMLPlatformUtils::Initialize (); @@ -316,7 +331,7 @@ validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, list dcp, shared_ptr reel_mxf, function progress) +verify_asset (shared_ptr dcp, shared_ptr reel_file_asset, function progress) { - string const actual_hash = reel_mxf->asset_ref()->hash(progress); + auto const actual_hash = reel_file_asset->asset_ref()->hash(progress); - list > pkls = dcp->pkls(); + auto pkls = dcp->pkls(); /* We've read this DCP in so it must have at least one PKL */ DCP_ASSERT (!pkls.empty()); - shared_ptr asset = reel_mxf->asset_ref().asset(); + auto asset = reel_file_asset->asset_ref().asset(); optional pkl_hash; - BOOST_FOREACH (shared_ptr i, pkls) { - pkl_hash = i->hash (reel_mxf->asset_ref()->id()); + for (auto i: pkls) { + pkl_hash = i->hash (reel_file_asset->asset_ref()->id()); if (pkl_hash) { break; } @@ -383,35 +396,35 @@ verify_asset (shared_ptr dcp, shared_ptr reel_mxf, fun DCP_ASSERT (pkl_hash); - optional cpl_hash = reel_mxf->hash(); + auto cpl_hash = reel_file_asset->hash(); if (cpl_hash && *cpl_hash != *pkl_hash) { - return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER; + return VerifyAssetResult::CPL_PKL_DIFFER; } if (actual_hash != *pkl_hash) { - return VERIFY_ASSET_RESULT_BAD; + return VerifyAssetResult::BAD; } - return VERIFY_ASSET_RESULT_GOOD; + return VerifyAssetResult::GOOD; } void -verify_language_tag (string tag, list& notes) +verify_language_tag (string tag, vector& notes) { try { - dcp::LanguageTag test (tag); - } catch (dcp::LanguageTagError &) { - notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::BAD_LANGUAGE, tag)); + LanguageTag test (tag); + } catch (LanguageTagError &) { + notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_LANGUAGE, tag}); } } -enum VerifyPictureAssetResult +enum class VerifyPictureAssetResult { - VERIFY_PICTURE_ASSET_RESULT_GOOD, - VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE, - VERIFY_PICTURE_ASSET_RESULT_BAD, + GOOD, + FRAME_NEARLY_TOO_LARGE, + BAD, }; @@ -430,16 +443,16 @@ biggest_frame_size (shared_ptr frame) template optional -verify_picture_asset_type (shared_ptr reel_mxf, function progress) +verify_picture_asset_type (shared_ptr reel_file_asset, function progress) { - shared_ptr asset = dynamic_pointer_cast(reel_mxf->asset_ref().asset()); + auto asset = dynamic_pointer_cast(reel_file_asset->asset_ref().asset()); if (!asset) { return optional(); } int biggest_frame = 0; - shared_ptr reader = asset->start_read (); - int64_t const duration = asset->intrinsic_duration (); + auto reader = asset->start_read (); + auto const duration = asset->intrinsic_duration (); for (int64_t i = 0; i < duration; ++i) { shared_ptr frame = reader->get_frame (i); biggest_frame = max(biggest_frame, biggest_frame_size(frame)); @@ -449,21 +462,21 @@ verify_picture_asset_type (shared_ptr reel_mxf, functionedit_rate().as_float())); static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float())); if (biggest_frame > max_frame) { - return VERIFY_PICTURE_ASSET_RESULT_BAD; + return VerifyPictureAssetResult::BAD; } else if (biggest_frame > risky_frame) { - return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE; + return VerifyPictureAssetResult::FRAME_NEARLY_TOO_LARGE; } - return VERIFY_PICTURE_ASSET_RESULT_GOOD; + return VerifyPictureAssetResult::GOOD; } static VerifyPictureAssetResult -verify_picture_asset (shared_ptr reel_mxf, function progress) +verify_picture_asset (shared_ptr reel_file_asset, function progress) { - optional r = verify_picture_asset_type(reel_mxf, progress); + auto r = verify_picture_asset_type(reel_file_asset, progress); if (!r) { - r = verify_picture_asset_type(reel_mxf, progress); + r = verify_picture_asset_type(reel_file_asset, progress); } DCP_ASSERT (r); @@ -477,47 +490,39 @@ verify_main_picture_asset ( shared_ptr reel_asset, function)> stage, function progress, - list& notes + vector& notes ) { - shared_ptr asset = reel_asset->asset(); - boost::filesystem::path const file = *asset->file(); + auto asset = reel_asset->asset(); + auto const file = *asset->file(); stage ("Checking picture asset hash", file); - VerifyAssetResult const r = verify_asset (dcp, reel_asset, progress); + auto const r = verify_asset (dcp, reel_asset, progress); switch (r) { - case VERIFY_ASSET_RESULT_BAD: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, file - ) - ); + case VerifyAssetResult::BAD: + notes.push_back ({ + VerificationNote::Type::ERROR, VerificationNote::Code::INCORRECT_PICTURE_HASH, file + }); break; - case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE, file - ) - ); + case VerifyAssetResult::CPL_PKL_DIFFER: + notes.push_back ({ + VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_PICTURE_HASHES, file + }); break; default: break; } stage ("Checking picture frame sizes", asset->file()); - VerifyPictureAssetResult const pr = verify_picture_asset (reel_asset, progress); + auto const pr = verify_picture_asset (reel_asset, progress); switch (pr) { - case VERIFY_PICTURE_ASSET_RESULT_BAD: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_FRAME_TOO_LARGE_IN_BYTES, file - ) - ); + case VerifyPictureAssetResult::BAD: + notes.push_back ({ + VerificationNote::Type::ERROR, VerificationNote::Code::INVALID_PICTURE_FRAME_SIZE_IN_BYTES, file + }); break; - case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_WARNING, VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE_IN_BYTES, file - ) - ); + case VerifyPictureAssetResult::FRAME_NEARLY_TOO_LARGE: + notes.push_back ({ + VerificationNote::Type::WARNING, VerificationNote::Code::NEARLY_INVALID_PICTURE_FRAME_SIZE_IN_BYTES, file + }); break; default: break; @@ -525,57 +530,50 @@ verify_main_picture_asset ( /* Only flat/scope allowed by Bv2.1 */ if ( - asset->size() != dcp::Size(2048, 858) && - asset->size() != dcp::Size(1998, 1080) && - asset->size() != dcp::Size(4096, 1716) && - asset->size() != dcp::Size(3996, 2160)) { - notes.push_back( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, - VerificationNote::PICTURE_ASSET_INVALID_SIZE_IN_PIXELS, - String::compose("%1x%2", asset->size().width, asset->size().height), - file - ) - ); + asset->size() != Size(2048, 858) && + asset->size() != Size(1998, 1080) && + asset->size() != Size(4096, 1716) && + asset->size() != Size(3996, 2160)) { + notes.push_back({ + VerificationNote::Type::BV21_ERROR, + VerificationNote::Code::INVALID_PICTURE_SIZE_IN_PIXELS, + String::compose("%1x%2", asset->size().width, asset->size().height), + file + }); } /* Only 24, 25, 48fps allowed for 2K */ if ( - (asset->size() == dcp::Size(2048, 858) || asset->size() == dcp::Size(1998, 1080)) && - (asset->edit_rate() != dcp::Fraction(24, 1) && asset->edit_rate() != dcp::Fraction(25, 1) && asset->edit_rate() != dcp::Fraction(48, 1)) + (asset->size() == Size(2048, 858) || asset->size() == Size(1998, 1080)) && + (asset->edit_rate() != Fraction(24, 1) && asset->edit_rate() != Fraction(25, 1) && asset->edit_rate() != Fraction(48, 1)) ) { - notes.push_back( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, - VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_2K, - String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), - file - ) - ); + notes.push_back({ + VerificationNote::Type::BV21_ERROR, + VerificationNote::Code::INVALID_PICTURE_FRAME_RATE_FOR_2K, + String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), + file + }); } - if (asset->size() == dcp::Size(4096, 1716) || asset->size() == dcp::Size(3996, 2160)) { + if (asset->size() == Size(4096, 1716) || asset->size() == Size(3996, 2160)) { /* Only 24fps allowed for 4K */ - if (asset->edit_rate() != dcp::Fraction(24, 1)) { - notes.push_back( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, - VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_4K, - String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), - file - ) - ); + if (asset->edit_rate() != Fraction(24, 1)) { + notes.push_back({ + VerificationNote::Type::BV21_ERROR, + VerificationNote::Code::INVALID_PICTURE_FRAME_RATE_FOR_4K, + String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), + file + }); } /* Only 2D allowed for 4K */ if (dynamic_pointer_cast(asset)) { - notes.push_back( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, - VerificationNote::PICTURE_ASSET_4K_3D, - file - ) - ); + notes.push_back({ + VerificationNote::Type::BV21_ERROR, + VerificationNote::Code::INVALID_PICTURE_ASSET_RESOLUTION_FOR_3D, + String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), + file + }); } } @@ -589,26 +587,18 @@ verify_main_sound_asset ( shared_ptr reel_asset, function)> stage, function progress, - list& notes + vector& notes ) { - shared_ptr asset = reel_asset->asset(); + auto asset = reel_asset->asset(); stage ("Checking sound asset hash", asset->file()); - VerifyAssetResult const r = verify_asset (dcp, reel_asset, progress); + auto const r = verify_asset (dcp, reel_asset, progress); switch (r) { - case VERIFY_ASSET_RESULT_BAD: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *asset->file() - ) - ); + case VerifyAssetResult::BAD: + notes.push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::INCORRECT_SOUND_HASH, *asset->file()}); break; - case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE, *asset->file() - ) - ); + case VerifyAssetResult::CPL_PKL_DIFFER: + notes.push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_SOUND_HASHES, *asset->file()}); break; default: break; @@ -617,35 +607,116 @@ verify_main_sound_asset ( stage ("Checking sound asset metadata", asset->file()); verify_language_tag (asset->language(), notes); + if (asset->sampling_rate() != 48000) { + notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_SOUND_FRAME_RATE, raw_convert(asset->sampling_rate()), *asset->file()}); + } } static void -verify_main_subtitle_reel (shared_ptr reel_asset, list& notes) +verify_main_subtitle_reel (shared_ptr reel_asset, vector& notes) { /* XXX: is Language compulsory? */ if (reel_asset->language()) { verify_language_tag (*reel_asset->language(), notes); } + + if (!reel_asset->entry_point()) { + notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_SUBTITLE_ENTRY_POINT, reel_asset->id() }); + } else if (reel_asset->entry_point().get()) { + notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INCORRECT_SUBTITLE_ENTRY_POINT, reel_asset->id() }); + } } static void -verify_closed_caption_reel (shared_ptr reel_asset, list& notes) +verify_closed_caption_reel (shared_ptr reel_asset, vector& notes) { /* XXX: is Language compulsory? */ if (reel_asset->language()) { verify_language_tag (*reel_asset->language(), notes); } + + if (!reel_asset->entry_point()) { + notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_CLOSED_CAPTION_ENTRY_POINT, reel_asset->id() }); + } else if (reel_asset->entry_point().get()) { + notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INCORRECT_CLOSED_CAPTION_ENTRY_POINT, reel_asset->id() }); + } } +struct State +{ + boost::optional subtitle_language; +}; + + +/** Verify stuff that is common to both subtitles and closed captions */ +void +verify_smpte_timed_text_asset ( + shared_ptr asset, + vector& notes + ) +{ + if (asset->language()) { + verify_language_tag (*asset->language(), notes); + } else { + notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_SUBTITLE_LANGUAGE, *asset->file() }); + } + + auto const size = boost::filesystem::file_size(asset->file().get()); + if (size > 115 * 1024 * 1024) { + notes.push_back ( + { VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_TIMED_TEXT_SIZE_IN_BYTES, raw_convert(size), *asset->file() } + ); + } + + /* XXX: I'm not sure what Bv2.1_7.2.1 means when it says "the font resource shall not be larger than 10MB" + * but I'm hoping that checking for the total size of all fonts being <= 10MB will do. + */ + auto fonts = asset->font_data (); + int total_size = 0; + for (auto i: fonts) { + total_size += i.second.size(); + } + if (total_size > 10 * 1024 * 1024) { + notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_TIMED_TEXT_FONT_SIZE_IN_BYTES, raw_convert(total_size), asset->file().get() }); + } + + if (!asset->start_time()) { + notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_SUBTITLE_START_TIME, asset->file().get() }); + } else if (asset->start_time() != Time()) { + notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_SUBTITLE_START_TIME, asset->file().get() }); + } +} + + +/** Verify SMPTE subtitle-only stuff */ +void +verify_smpte_subtitle_asset ( + shared_ptr asset, + vector& notes, + State& state + ) +{ + if (asset->language()) { + if (!state.subtitle_language) { + state.subtitle_language = *asset->language(); + } else if (state.subtitle_language != *asset->language()) { + notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISMATCHED_SUBTITLE_LANGUAGES }); + } + } +} + + +/** Verify all subtitle stuff */ static void verify_subtitle_asset ( shared_ptr asset, function)> stage, boost::filesystem::path xsd_dtd_directory, - list& notes + vector& notes, + State& state ) { stage ("Checking subtitle XML", asset->file()); @@ -654,64 +725,381 @@ verify_subtitle_asset ( */ validate_xml (asset->raw_xml(), xsd_dtd_directory, notes); - shared_ptr smpte = dynamic_pointer_cast(asset); + auto smpte = dynamic_pointer_cast(asset); if (smpte) { - if (smpte->language()) { - verify_language_tag (*smpte->language(), notes); - } else { - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_LANGUAGE, *asset->file() - ) - ); - } - if (boost::filesystem::file_size(*asset->file()) > 115 * 1024 * 1024) { - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, VerificationNote::TIMED_TEXT_ASSET_TOO_LARGE_IN_BYTES, *asset->file() - ) - ); - } - /* XXX: I'm not sure what Bv2.1_7.2.1 means when it says "the font resource shall not be larger than 10MB" - * but I'm hoping that checking for the total size of all fonts being <= 10MB will do. - */ - map fonts = asset->font_data (); - int total_size = 0; - for (map::const_iterator i = fonts.begin(); i != fonts.end(); ++i) { - total_size += i->second.size(); - } - if (total_size > 10 * 1024 * 1024) { - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, VerificationNote::TIMED_TEXT_FONTS_TOO_LARGE_IN_BYTES, *asset->file() - ) - ); - } + verify_smpte_timed_text_asset (smpte, notes); + verify_smpte_subtitle_asset (smpte, notes, state); } } +/** Verify all closed caption stuff */ static void verify_closed_caption_asset ( shared_ptr asset, function)> stage, boost::filesystem::path xsd_dtd_directory, - list& notes + vector& notes ) { - verify_subtitle_asset (asset, stage, xsd_dtd_directory, notes); + stage ("Checking closed caption XML", asset->file()); + /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk + * gets passed through libdcp which may clean up and therefore hide errors. + */ + validate_xml (asset->raw_xml(), xsd_dtd_directory, notes); + + auto smpte = dynamic_pointer_cast(asset); + if (smpte) { + verify_smpte_timed_text_asset (smpte, notes); + } if (asset->raw_xml().size() > 256 * 1024) { - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, VerificationNote::CLOSED_CAPTION_XML_TOO_LARGE_IN_BYTES, *asset->file() - ) - ); + notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_CLOSED_CAPTION_XML_SIZE_IN_BYTES, raw_convert(asset->raw_xml().size()), *asset->file()}); + } +} + + +static +void +verify_text_timing ( + vector> reels, + int edit_rate, + vector& notes, + std::function)> check, + std::function)> xml, + std::function)> duration + ) +{ + /* end of last subtitle (in editable units) */ + optional last_out; + auto too_short = false; + auto too_close = false; + auto too_early = false; + auto reel_overlap = false; + /* current reel start time (in editable units) */ + int64_t reel_offset = 0; + + std::function, optional