2 Copyright (C) 2018-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of libdcp.
6 libdcp is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 libdcp is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with libdcp. If not, see <http://www.gnu.org/licenses/>.
19 In addition, as a special exception, the copyright holders give
20 permission to link the code of portions of this program with the
21 OpenSSL library under certain conditions as described in each
22 individual source file, and distribute linked combinations
25 You must obey the GNU General Public License in all respects
26 for all of the code used other than OpenSSL. If you modify
27 file(s) with this exception, you may extend this exception to your
28 version of the file(s), but you are not obligated to do so. If you
29 do not wish to do so, delete this exception statement from your
30 version. If you delete this exception statement from all source
31 files in the program, then also delete it here.
35 /** @file src/verify.cc
36 * @brief dcp::verify() method and associated code
40 #include "compose.hpp"
43 #include "exceptions.h"
44 #include "interop_subtitle_asset.h"
45 #include "mono_picture_asset.h"
46 #include "mono_picture_frame.h"
47 #include "raw_convert.h"
49 #include "reel_closed_caption_asset.h"
50 #include "reel_interop_subtitle_asset.h"
51 #include "reel_markers_asset.h"
52 #include "reel_picture_asset.h"
53 #include "reel_sound_asset.h"
54 #include "reel_smpte_subtitle_asset.h"
55 #include "reel_subtitle_asset.h"
56 #include "smpte_subtitle_asset.h"
57 #include "stereo_picture_asset.h"
58 #include "stereo_picture_frame.h"
60 #include "verify_j2k.h"
61 #include <xercesc/dom/DOMAttr.hpp>
62 #include <xercesc/dom/DOMDocument.hpp>
63 #include <xercesc/dom/DOMError.hpp>
64 #include <xercesc/dom/DOMErrorHandler.hpp>
65 #include <xercesc/dom/DOMException.hpp>
66 #include <xercesc/dom/DOMImplementation.hpp>
67 #include <xercesc/dom/DOMImplementationLS.hpp>
68 #include <xercesc/dom/DOMImplementationRegistry.hpp>
69 #include <xercesc/dom/DOMLSParser.hpp>
70 #include <xercesc/dom/DOMLocator.hpp>
71 #include <xercesc/dom/DOMNamedNodeMap.hpp>
72 #include <xercesc/dom/DOMNodeList.hpp>
73 #include <xercesc/framework/LocalFileInputSource.hpp>
74 #include <xercesc/framework/MemBufInputSource.hpp>
75 #include <xercesc/parsers/AbstractDOMParser.hpp>
76 #include <xercesc/parsers/XercesDOMParser.hpp>
77 #include <xercesc/sax/HandlerBase.hpp>
78 #include <xercesc/util/PlatformUtils.hpp>
79 #include <boost/algorithm/string.hpp>
91 using std::shared_ptr;
92 using std::make_shared;
93 using boost::optional;
94 using boost::function;
95 using std::dynamic_pointer_cast;
99 using namespace xercesc;
104 xml_ch_to_string (XMLCh const * a)
106 char* x = XMLString::transcode(a);
108 XMLString::release(&x);
113 class XMLValidationError
116 XMLValidationError (SAXParseException const & e)
117 : _message (xml_ch_to_string(e.getMessage()))
118 , _line (e.getLineNumber())
119 , _column (e.getColumnNumber())
120 , _public_id (e.getPublicId() ? xml_ch_to_string(e.getPublicId()) : "")
121 , _system_id (e.getSystemId() ? xml_ch_to_string(e.getSystemId()) : "")
126 string message () const {
130 uint64_t line () const {
134 uint64_t column () const {
138 string public_id () const {
142 string system_id () const {
155 class DCPErrorHandler : public ErrorHandler
158 void warning(const SAXParseException& e) override
160 maybe_add (XMLValidationError(e));
163 void error(const SAXParseException& e) override
165 maybe_add (XMLValidationError(e));
168 void fatalError(const SAXParseException& e) override
170 maybe_add (XMLValidationError(e));
173 void resetErrors() override {
177 list<XMLValidationError> errors () const {
182 void maybe_add (XMLValidationError e)
184 /* XXX: nasty hack */
186 e.message().find("schema document") != string::npos &&
187 e.message().find("has different target namespace from the one specified in instance document") != string::npos
192 _errors.push_back (e);
195 list<XMLValidationError> _errors;
202 StringToXMLCh (string a)
204 _buffer = XMLString::transcode(a.c_str());
207 StringToXMLCh (StringToXMLCh const&) = delete;
208 StringToXMLCh& operator= (StringToXMLCh const&) = delete;
212 XMLString::release (&_buffer);
215 XMLCh const * get () const {
224 class LocalFileResolver : public EntityResolver
227 LocalFileResolver (boost::filesystem::path xsd_dtd_directory)
228 : _xsd_dtd_directory (xsd_dtd_directory)
230 /* XXX: I'm not clear on what things need to be in this list; some XSDs are apparently, magically
231 * found without being here.
233 add("http://www.w3.org/2001/XMLSchema.dtd", "XMLSchema.dtd");
234 add("http://www.w3.org/2001/03/xml.xsd", "xml.xsd");
235 add("http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd", "xmldsig-core-schema.xsd");
236 add("http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd", "Main-Stereo-Picture-CPL.xsd");
237 add("http://www.digicine.com/PROTO-ASDCP-CPL-20040511.xsd", "PROTO-ASDCP-CPL-20040511.xsd");
238 add("http://www.digicine.com/PROTO-ASDCP-PKL-20040311.xsd", "PROTO-ASDCP-PKL-20040311.xsd");
239 add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd");
240 add("http://www.digicine.com/PROTO-ASDCP-CC-CPL-20070926#", "PROTO-ASDCP-CC-CPL-20070926.xsd");
241 add("interop-subs", "DCSubtitle.v1.mattsson.xsd");
242 add("http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd", "SMPTE-428-7-2010-DCST.xsd");
243 add("http://www.smpte-ra.org/schemas/429-16/2014/CPL-Metadata", "SMPTE-429-16.xsd");
244 add("http://www.dolby.com/schemas/2012/AD", "Dolby-2012-AD.xsd");
245 add("http://www.smpte-ra.org/schemas/429-10/2008/Main-Stereo-Picture-CPL", "SMPTE-429-10-2008.xsd");
248 InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id) override
253 auto system_id_str = xml_ch_to_string (system_id);
254 auto p = _xsd_dtd_directory;
255 if (_files.find(system_id_str) == _files.end()) {
258 p /= _files[system_id_str];
260 StringToXMLCh ch (p.string());
261 return new LocalFileInputSource(ch.get());
265 void add (string uri, string file)
270 std::map<string, string> _files;
271 boost::filesystem::path _xsd_dtd_directory;
276 parse (XercesDOMParser& parser, boost::filesystem::path xml)
278 parser.parse(xml.string().c_str());
283 parse (XercesDOMParser& parser, string xml)
285 xercesc::MemBufInputSource buf(reinterpret_cast<unsigned char const*>(xml.c_str()), xml.size(), "");
292 validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, vector<VerificationNote>& notes)
295 XMLPlatformUtils::Initialize ();
296 } catch (XMLException& e) {
297 throw MiscError ("Failed to initialise xerces library");
300 DCPErrorHandler error_handler;
302 /* All the xerces objects in this scope must be destroyed before XMLPlatformUtils::Terminate() is called */
304 XercesDOMParser parser;
305 parser.setValidationScheme(XercesDOMParser::Val_Always);
306 parser.setDoNamespaces(true);
307 parser.setDoSchema(true);
309 vector<string> schema;
310 schema.push_back("xml.xsd");
311 schema.push_back("xmldsig-core-schema.xsd");
312 schema.push_back("SMPTE-429-7-2006-CPL.xsd");
313 schema.push_back("SMPTE-429-8-2006-PKL.xsd");
314 schema.push_back("SMPTE-429-9-2007-AM.xsd");
315 schema.push_back("Main-Stereo-Picture-CPL.xsd");
316 schema.push_back("PROTO-ASDCP-CPL-20040511.xsd");
317 schema.push_back("PROTO-ASDCP-PKL-20040311.xsd");
318 schema.push_back("PROTO-ASDCP-AM-20040311.xsd");
319 schema.push_back("DCSubtitle.v1.mattsson.xsd");
320 schema.push_back("DCDMSubtitle-2010.xsd");
321 schema.push_back("PROTO-ASDCP-CC-CPL-20070926.xsd");
322 schema.push_back("SMPTE-429-16.xsd");
323 schema.push_back("Dolby-2012-AD.xsd");
324 schema.push_back("SMPTE-429-10-2008.xsd");
325 schema.push_back("xlink.xsd");
326 schema.push_back("SMPTE-335-2012.xsd");
327 schema.push_back("SMPTE-395-2014-13-1-aaf.xsd");
328 schema.push_back("isdcf-mca.xsd");
329 schema.push_back("SMPTE-429-12-2008.xsd");
331 /* XXX: I'm not especially clear what this is for, but it seems to be necessary.
332 * Schemas that are not mentioned in this list are not read, and the things
333 * they describe are not checked.
336 for (auto i: schema) {
337 locations += String::compose("%1 %1 ", i, i);
340 parser.setExternalSchemaLocation(locations.c_str());
341 parser.setValidationSchemaFullChecking(true);
342 parser.setErrorHandler(&error_handler);
344 LocalFileResolver resolver (xsd_dtd_directory);
345 parser.setEntityResolver(&resolver);
348 parser.resetDocumentPool();
350 } catch (XMLException& e) {
351 throw MiscError(xml_ch_to_string(e.getMessage()));
352 } catch (DOMException& e) {
353 throw MiscError(xml_ch_to_string(e.getMessage()));
355 throw MiscError("Unknown exception from xerces");
359 XMLPlatformUtils::Terminate ();
361 for (auto i: error_handler.errors()) {
363 VerificationNote::Type::ERROR,
364 VerificationNote::Code::INVALID_XML,
366 boost::trim_copy(i.public_id() + " " + i.system_id()),
373 enum class VerifyAssetResult {
380 static VerifyAssetResult
381 verify_asset (shared_ptr<const DCP> dcp, shared_ptr<const ReelFileAsset> reel_file_asset, function<void (float)> progress)
383 auto const actual_hash = reel_file_asset->asset_ref()->hash(progress);
385 auto pkls = dcp->pkls();
386 /* We've read this DCP in so it must have at least one PKL */
387 DCP_ASSERT (!pkls.empty());
389 auto asset = reel_file_asset->asset_ref().asset();
391 optional<string> pkl_hash;
393 pkl_hash = i->hash (reel_file_asset->asset_ref()->id());
399 DCP_ASSERT (pkl_hash);
401 auto cpl_hash = reel_file_asset->hash();
402 if (cpl_hash && *cpl_hash != *pkl_hash) {
403 return VerifyAssetResult::CPL_PKL_DIFFER;
406 if (actual_hash != *pkl_hash) {
407 return VerifyAssetResult::BAD;
410 return VerifyAssetResult::GOOD;
415 verify_language_tag (string tag, vector<VerificationNote>& notes)
418 LanguageTag test (tag);
419 } catch (LanguageTagError &) {
420 notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_LANGUAGE, tag});
426 verify_picture_asset (shared_ptr<const ReelFileAsset> reel_file_asset, boost::filesystem::path file, vector<VerificationNote>& notes, function<void (float)> progress)
428 int biggest_frame = 0;
429 auto asset = dynamic_pointer_cast<PictureAsset>(reel_file_asset->asset_ref().asset());
430 auto const duration = asset->intrinsic_duration ();
432 auto check_and_add = [¬es](vector<VerificationNote> const& j2k_notes) {
433 for (auto i: j2k_notes) {
434 if (find(notes.begin(), notes.end(), i) == notes.end()) {
440 if (auto mono_asset = dynamic_pointer_cast<MonoPictureAsset>(reel_file_asset->asset_ref().asset())) {
441 auto reader = mono_asset->start_read ();
442 for (int64_t i = 0; i < duration; ++i) {
443 auto frame = reader->get_frame (i);
444 biggest_frame = max(biggest_frame, frame->size());
445 if (!mono_asset->encrypted() || mono_asset->key()) {
446 vector<VerificationNote> j2k_notes;
447 verify_j2k (frame, j2k_notes);
448 check_and_add (j2k_notes);
450 progress (float(i) / duration);
452 } else if (auto stereo_asset = dynamic_pointer_cast<StereoPictureAsset>(asset)) {
453 auto reader = stereo_asset->start_read ();
454 for (int64_t i = 0; i < duration; ++i) {
455 auto frame = reader->get_frame (i);
456 biggest_frame = max(biggest_frame, max(frame->left()->size(), frame->right()->size()));
457 if (!stereo_asset->encrypted() || stereo_asset->key()) {
458 vector<VerificationNote> j2k_notes;
459 verify_j2k (frame->left(), j2k_notes);
460 verify_j2k (frame->right(), j2k_notes);
461 check_and_add (j2k_notes);
463 progress (float(i) / duration);
468 static const int max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float()));
469 static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float()));
470 if (biggest_frame > max_frame) {
472 VerificationNote::Type::ERROR, VerificationNote::Code::INVALID_PICTURE_FRAME_SIZE_IN_BYTES, file
474 } else if (biggest_frame > risky_frame) {
476 VerificationNote::Type::WARNING, VerificationNote::Code::NEARLY_INVALID_PICTURE_FRAME_SIZE_IN_BYTES, file
483 verify_main_picture_asset (
484 shared_ptr<const DCP> dcp,
485 shared_ptr<const ReelPictureAsset> reel_asset,
486 function<void (string, optional<boost::filesystem::path>)> stage,
487 function<void (float)> progress,
488 vector<VerificationNote>& notes
491 auto asset = reel_asset->asset();
492 auto const file = *asset->file();
493 stage ("Checking picture asset hash", file);
494 auto const r = verify_asset (dcp, reel_asset, progress);
496 case VerifyAssetResult::BAD:
498 VerificationNote::Type::ERROR, VerificationNote::Code::INCORRECT_PICTURE_HASH, file
501 case VerifyAssetResult::CPL_PKL_DIFFER:
503 VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_PICTURE_HASHES, file
509 stage ("Checking picture frame sizes", asset->file());
510 verify_picture_asset (reel_asset, file, notes, progress);
512 /* Only flat/scope allowed by Bv2.1 */
514 asset->size() != Size(2048, 858) &&
515 asset->size() != Size(1998, 1080) &&
516 asset->size() != Size(4096, 1716) &&
517 asset->size() != Size(3996, 2160)) {
519 VerificationNote::Type::BV21_ERROR,
520 VerificationNote::Code::INVALID_PICTURE_SIZE_IN_PIXELS,
521 String::compose("%1x%2", asset->size().width, asset->size().height),
526 /* Only 24, 25, 48fps allowed for 2K */
528 (asset->size() == Size(2048, 858) || asset->size() == Size(1998, 1080)) &&
529 (asset->edit_rate() != Fraction(24, 1) && asset->edit_rate() != Fraction(25, 1) && asset->edit_rate() != Fraction(48, 1))
532 VerificationNote::Type::BV21_ERROR,
533 VerificationNote::Code::INVALID_PICTURE_FRAME_RATE_FOR_2K,
534 String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator),
539 if (asset->size() == Size(4096, 1716) || asset->size() == Size(3996, 2160)) {
540 /* Only 24fps allowed for 4K */
541 if (asset->edit_rate() != Fraction(24, 1)) {
543 VerificationNote::Type::BV21_ERROR,
544 VerificationNote::Code::INVALID_PICTURE_FRAME_RATE_FOR_4K,
545 String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator),
550 /* Only 2D allowed for 4K */
551 if (dynamic_pointer_cast<const StereoPictureAsset>(asset)) {
553 VerificationNote::Type::BV21_ERROR,
554 VerificationNote::Code::INVALID_PICTURE_ASSET_RESOLUTION_FOR_3D,
555 String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator),
566 verify_main_sound_asset (
567 shared_ptr<const DCP> dcp,
568 shared_ptr<const ReelSoundAsset> reel_asset,
569 function<void (string, optional<boost::filesystem::path>)> stage,
570 function<void (float)> progress,
571 vector<VerificationNote>& notes
574 auto asset = reel_asset->asset();
575 stage ("Checking sound asset hash", asset->file());
576 auto const r = verify_asset (dcp, reel_asset, progress);
578 case VerifyAssetResult::BAD:
579 notes.push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::INCORRECT_SOUND_HASH, *asset->file()});
581 case VerifyAssetResult::CPL_PKL_DIFFER:
582 notes.push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_SOUND_HASHES, *asset->file()});
588 stage ("Checking sound asset metadata", asset->file());
590 if (auto lang = asset->language()) {
591 verify_language_tag (*lang, notes);
593 if (asset->sampling_rate() != 48000) {
594 notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_SOUND_FRAME_RATE, raw_convert<string>(asset->sampling_rate()), *asset->file()});
600 verify_main_subtitle_reel (shared_ptr<const ReelSubtitleAsset> reel_asset, vector<VerificationNote>& notes)
602 /* XXX: is Language compulsory? */
603 if (reel_asset->language()) {
604 verify_language_tag (*reel_asset->language(), notes);
607 if (!reel_asset->entry_point()) {
608 notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_SUBTITLE_ENTRY_POINT, reel_asset->id() });
609 } else if (reel_asset->entry_point().get()) {
610 notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INCORRECT_SUBTITLE_ENTRY_POINT, reel_asset->id() });
616 verify_closed_caption_reel (shared_ptr<const ReelClosedCaptionAsset> reel_asset, vector<VerificationNote>& notes)
618 /* XXX: is Language compulsory? */
619 if (reel_asset->language()) {
620 verify_language_tag (*reel_asset->language(), notes);
623 if (!reel_asset->entry_point()) {
624 notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_CLOSED_CAPTION_ENTRY_POINT, reel_asset->id() });
625 } else if (reel_asset->entry_point().get()) {
626 notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INCORRECT_CLOSED_CAPTION_ENTRY_POINT, reel_asset->id() });
633 boost::optional<string> subtitle_language;
637 /** Verify stuff that is common to both subtitles and closed captions */
639 verify_smpte_timed_text_asset (
640 shared_ptr<const SMPTESubtitleAsset> asset,
641 optional<int64_t> reel_asset_duration,
642 vector<VerificationNote>& notes
645 if (asset->language()) {
646 verify_language_tag (*asset->language(), notes);
648 notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_SUBTITLE_LANGUAGE, *asset->file() });
651 auto const size = boost::filesystem::file_size(asset->file().get());
652 if (size > 115 * 1024 * 1024) {
654 { VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_TIMED_TEXT_SIZE_IN_BYTES, raw_convert<string>(size), *asset->file() }
658 /* XXX: I'm not sure what Bv2.1_7.2.1 means when it says "the font resource shall not be larger than 10MB"
659 * but I'm hoping that checking for the total size of all fonts being <= 10MB will do.
661 auto fonts = asset->font_data ();
663 for (auto i: fonts) {
664 total_size += i.second.size();
666 if (total_size > 10 * 1024 * 1024) {
667 notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_TIMED_TEXT_FONT_SIZE_IN_BYTES, raw_convert<string>(total_size), asset->file().get() });
670 if (!asset->start_time()) {
671 notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_SUBTITLE_START_TIME, asset->file().get() });
672 } else if (asset->start_time() != Time()) {
673 notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_SUBTITLE_START_TIME, asset->file().get() });
676 if (reel_asset_duration && *reel_asset_duration != asset->intrinsic_duration()) {
679 VerificationNote::Type::BV21_ERROR,
680 VerificationNote::Code::MISMATCHED_TIMED_TEXT_DURATION,
681 String::compose("%1 %2", *reel_asset_duration, asset->intrinsic_duration()),
688 /** Verify SMPTE subtitle-only stuff */
690 verify_smpte_subtitle_asset (
691 shared_ptr<const SMPTESubtitleAsset> asset,
692 vector<VerificationNote>& notes,
696 if (asset->language()) {
697 if (!state.subtitle_language) {
698 state.subtitle_language = *asset->language();
699 } else if (state.subtitle_language != *asset->language()) {
700 notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISMATCHED_SUBTITLE_LANGUAGES });
704 DCP_ASSERT (asset->resource_id());
705 auto xml_id = asset->xml_id();
707 if (asset->resource_id().get() != xml_id) {
708 notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISMATCHED_TIMED_TEXT_RESOURCE_ID });
711 if (asset->id() == asset->resource_id().get() || asset->id() == xml_id) {
712 notes.push_back ({ VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INCORRECT_TIMED_TEXT_ASSET_ID });
715 notes.push_back ({VerificationNote::Type::WARNING, VerificationNote::Code::MISSED_CHECK_OF_ENCRYPTED});
720 /** Verify all subtitle stuff */
722 verify_subtitle_asset (
723 shared_ptr<const SubtitleAsset> asset,
724 optional<int64_t> reel_asset_duration,
725 function<void (string, optional<boost::filesystem::path>)> stage,
726 boost::filesystem::path xsd_dtd_directory,
727 vector<VerificationNote>& notes,
731 stage ("Checking subtitle XML", asset->file());
732 /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk
733 * gets passed through libdcp which may clean up and therefore hide errors.
735 if (asset->raw_xml()) {
736 validate_xml (asset->raw_xml().get(), xsd_dtd_directory, notes);
738 notes.push_back ({VerificationNote::Type::WARNING, VerificationNote::Code::MISSED_CHECK_OF_ENCRYPTED});
741 auto smpte = dynamic_pointer_cast<const SMPTESubtitleAsset>(asset);
743 verify_smpte_timed_text_asset (smpte, reel_asset_duration, notes);
744 verify_smpte_subtitle_asset (smpte, notes, state);
749 /** Verify all closed caption stuff */
751 verify_closed_caption_asset (
752 shared_ptr<const SubtitleAsset> asset,
753 optional<int64_t> reel_asset_duration,
754 function<void (string, optional<boost::filesystem::path>)> stage,
755 boost::filesystem::path xsd_dtd_directory,
756 vector<VerificationNote>& notes
759 stage ("Checking closed caption XML", asset->file());
760 /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk
761 * gets passed through libdcp which may clean up and therefore hide errors.
763 auto raw_xml = asset->raw_xml();
765 validate_xml (*raw_xml, xsd_dtd_directory, notes);
766 if (raw_xml->size() > 256 * 1024) {
767 notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_CLOSED_CAPTION_XML_SIZE_IN_BYTES, raw_convert<string>(raw_xml->size()), *asset->file()});
770 notes.push_back ({VerificationNote::Type::WARNING, VerificationNote::Code::MISSED_CHECK_OF_ENCRYPTED});
773 auto smpte = dynamic_pointer_cast<const SMPTESubtitleAsset>(asset);
775 verify_smpte_timed_text_asset (smpte, reel_asset_duration, notes);
780 /** Check the timing of the individual subtitles and make sure there are no empty <Text> nodes */
783 verify_text_details (
784 vector<shared_ptr<Reel>> reels,
786 vector<VerificationNote>& notes,
787 std::function<bool (shared_ptr<Reel>)> check,
788 std::function<optional<string> (shared_ptr<Reel>)> xml,
789 std::function<int64_t (shared_ptr<Reel>)> duration
792 /* end of last subtitle (in editable units) */
793 optional<int64_t> last_out;
794 auto too_short = false;
795 auto too_close = false;
796 auto too_early = false;
797 auto reel_overlap = false;
798 auto empty_text = false;
799 /* current reel start time (in editable units) */
800 int64_t reel_offset = 0;
802 std::function<void (cxml::ConstNodePtr, optional<int>, optional<Time>, int, bool)> parse;
803 parse = [&parse, &last_out, &too_short, &too_close, &too_early, &empty_text, &reel_offset](cxml::ConstNodePtr node, optional<int> tcr, optional<Time> start_time, int er, bool first_reel) {
804 if (node->name() == "Subtitle") {
805 Time in (node->string_attribute("TimeIn"), tcr);
809 Time out (node->string_attribute("TimeOut"), tcr);
813 if (first_reel && tcr && in < Time(0, 0, 4, 0, *tcr)) {
816 auto length = out - in;
817 if (length.as_editable_units_ceil(er) < 15) {
821 /* XXX: this feels dubious - is it really what Bv2.1 means? */
822 auto distance = reel_offset + in.as_editable_units_ceil(er) - *last_out;
823 if (distance >= 0 && distance < 2) {
827 last_out = reel_offset + out.as_editable_units_floor(er);
828 } else if (node->name() == "Text") {
829 std::function<bool (cxml::ConstNodePtr)> node_has_content = [&](cxml::ConstNodePtr node) {
830 if (!node->content().empty()) {
833 for (auto i: node->node_children()) {
834 if (node_has_content(i)) {
840 if (!node_has_content(node)) {
845 for (auto i: node->node_children()) {
846 parse(i, tcr, start_time, er, first_reel);
850 for (auto i = 0U; i < reels.size(); ++i) {
851 if (!check(reels[i])) {
855 auto reel_xml = xml(reels[i]);
857 notes.push_back ({VerificationNote::Type::WARNING, VerificationNote::Code::MISSED_CHECK_OF_ENCRYPTED});
861 /* We need to look at <Subtitle> instances in the XML being checked, so we can't use the subtitles
862 * read in by libdcp's parser.
865 shared_ptr<cxml::Document> doc;
867 optional<Time> start_time;
869 doc = make_shared<cxml::Document>("SubtitleReel");
870 doc->read_string (*reel_xml);
871 tcr = doc->number_child<int>("TimeCodeRate");
872 auto start_time_string = doc->optional_string_child("StartTime");
873 if (start_time_string) {
874 start_time = Time(*start_time_string, tcr);
877 doc = make_shared<cxml::Document>("DCSubtitle");
878 doc->read_string (*reel_xml);
880 parse (doc, tcr, start_time, edit_rate, i == 0);
881 auto end = reel_offset + duration(reels[i]);
882 if (last_out && *last_out > end) {
888 if (last_out && *last_out > reel_offset) {
894 VerificationNote::Type::WARNING, VerificationNote::Code::INVALID_SUBTITLE_FIRST_TEXT_TIME
900 VerificationNote::Type::WARNING, VerificationNote::Code::INVALID_SUBTITLE_DURATION
906 VerificationNote::Type::WARNING, VerificationNote::Code::INVALID_SUBTITLE_SPACING
912 VerificationNote::Type::ERROR, VerificationNote::Code::SUBTITLE_OVERLAPS_REEL_BOUNDARY
918 VerificationNote::Type::WARNING, VerificationNote::Code::EMPTY_TEXT
926 verify_closed_caption_details (
927 vector<shared_ptr<Reel>> reels,
928 vector<VerificationNote>& notes
931 std::function<void (cxml::ConstNodePtr node, std::vector<cxml::ConstNodePtr>& text_or_image)> find_text_or_image;
932 find_text_or_image = [&find_text_or_image](cxml::ConstNodePtr node, std::vector<cxml::ConstNodePtr>& text_or_image) {
933 for (auto i: node->node_children()) {
934 if (i->name() == "Text") {
935 text_or_image.push_back (i);
937 find_text_or_image (i, text_or_image);
942 auto mismatched_valign = false;
943 auto incorrect_order = false;
945 std::function<void (cxml::ConstNodePtr)> parse;
946 parse = [&parse, &find_text_or_image, &mismatched_valign, &incorrect_order](cxml::ConstNodePtr node) {
947 if (node->name() == "Subtitle") {
948 vector<cxml::ConstNodePtr> text_or_image;
949 find_text_or_image (node, text_or_image);
950 optional<string> last_valign;
951 optional<float> last_vpos;
952 for (auto i: text_or_image) {
953 auto valign = i->optional_string_attribute("VAlign");
955 valign = i->optional_string_attribute("Valign").get_value_or("center");
957 auto vpos = i->optional_number_attribute<float>("VPosition");
959 vpos = i->optional_number_attribute<float>("Vposition").get_value_or(50);
963 if (*last_valign != valign) {
964 mismatched_valign = true;
967 last_valign = valign;
969 if (!mismatched_valign) {
971 if (*last_valign == "top" || *last_valign == "center") {
972 if (*vpos < *last_vpos) {
973 incorrect_order = true;
976 if (*vpos > *last_vpos) {
977 incorrect_order = true;
986 for (auto i: node->node_children()) {
991 for (auto reel: reels) {
992 for (auto ccap: reel->closed_captions()) {
993 auto reel_xml = ccap->asset()->raw_xml();
995 notes.push_back ({VerificationNote::Type::WARNING, VerificationNote::Code::MISSED_CHECK_OF_ENCRYPTED});
999 /* We need to look at <Subtitle> instances in the XML being checked, so we can't use the subtitles
1000 * read in by libdcp's parser.
1003 shared_ptr<cxml::Document> doc;
1005 optional<Time> start_time;
1007 doc = make_shared<cxml::Document>("SubtitleReel");
1008 doc->read_string (*reel_xml);
1010 doc = make_shared<cxml::Document>("DCSubtitle");
1011 doc->read_string (*reel_xml);
1017 if (mismatched_valign) {
1019 VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_CLOSED_CAPTION_VALIGN,
1023 if (incorrect_order) {
1025 VerificationNote::Type::ERROR, VerificationNote::Code::INCORRECT_CLOSED_CAPTION_ORDERING,
1031 struct LinesCharactersResult
1033 bool warning_length_exceeded = false;
1034 bool error_length_exceeded = false;
1035 bool line_count_exceeded = false;
1041 verify_text_lines_and_characters (
1042 shared_ptr<SubtitleAsset> asset,
1045 LinesCharactersResult* result
1051 Event (Time time_, float position_, int characters_)
1053 , position (position_)
1054 , characters (characters_)
1057 Event (Time time_, shared_ptr<Event> start_)
1063 int position; //< position from 0 at top of screen to 100 at bottom
1065 shared_ptr<Event> start;
1068 vector<shared_ptr<Event>> events;
1070 auto position = [](shared_ptr<const SubtitleString> sub) {
1071 switch (sub->v_align()) {
1073 return lrintf(sub->v_position() * 100);
1074 case VAlign::CENTER:
1075 return lrintf((0.5f + sub->v_position()) * 100);
1076 case VAlign::BOTTOM:
1077 return lrintf((1.0f - sub->v_position()) * 100);
1083 for (auto j: asset->subtitles()) {
1084 auto text = dynamic_pointer_cast<const SubtitleString>(j);
1086 auto in = make_shared<Event>(text->in(), position(text), text->text().length());
1087 events.push_back(in);
1088 events.push_back(make_shared<Event>(text->out(), in));
1092 std::sort(events.begin(), events.end(), [](shared_ptr<Event> const& a, shared_ptr<Event>const& b) {
1093 return a->time < b->time;
1096 map<int, int> current;
1097 for (auto i: events) {
1098 if (current.size() > 3) {
1099 result->line_count_exceeded = true;
1101 for (auto j: current) {
1102 if (j.second > warning_length) {
1103 result->warning_length_exceeded = true;
1105 if (j.second > error_length) {
1106 result->error_length_exceeded = true;
1111 /* end of a subtitle */
1112 DCP_ASSERT (current.find(i->start->position) != current.end());
1113 if (current[i->start->position] == i->start->characters) {
1114 current.erase(i->start->position);
1116 current[i->start->position] -= i->start->characters;
1119 /* start of a subtitle */
1120 if (current.find(i->position) == current.end()) {
1121 current[i->position] = i->characters;
1123 current[i->position] += i->characters;
1132 verify_text_details (vector<shared_ptr<Reel>> reels, vector<VerificationNote>& notes)
1134 if (reels.empty()) {
1138 if (reels[0]->main_subtitle()) {
1139 verify_text_details (reels, reels[0]->main_subtitle()->edit_rate().numerator, notes,
1140 [](shared_ptr<Reel> reel) {
1141 return static_cast<bool>(reel->main_subtitle());
1143 [](shared_ptr<Reel> reel) {
1144 auto interop = dynamic_pointer_cast<ReelInteropSubtitleAsset>(reel->main_subtitle());
1146 return interop->asset()->raw_xml();
1148 auto smpte = dynamic_pointer_cast<ReelSMPTESubtitleAsset>(reel->main_subtitle());
1150 return smpte->asset()->raw_xml();
1152 [](shared_ptr<Reel> reel) {
1153 return reel->main_subtitle()->actual_duration();
1158 for (auto i = 0U; i < reels[0]->closed_captions().size(); ++i) {
1159 verify_text_details (reels, reels[0]->closed_captions()[i]->edit_rate().numerator, notes,
1160 [i](shared_ptr<Reel> reel) {
1161 return i < reel->closed_captions().size();
1163 [i](shared_ptr<Reel> reel) {
1164 return reel->closed_captions()[i]->asset()->raw_xml();
1166 [i](shared_ptr<Reel> reel) {
1167 return reel->closed_captions()[i]->actual_duration();
1172 verify_closed_caption_details (reels, notes);
1177 verify_extension_metadata(shared_ptr<const CPL> cpl, vector<VerificationNote>& notes)
1179 DCP_ASSERT (cpl->file());
1180 cxml::Document doc ("CompositionPlaylist");
1181 doc.read_file (cpl->file().get());
1183 auto missing = false;
1186 if (auto reel_list = doc.node_child("ReelList")) {
1187 auto reels = reel_list->node_children("Reel");
1188 if (!reels.empty()) {
1189 if (auto asset_list = reels[0]->optional_node_child("AssetList")) {
1190 if (auto metadata = asset_list->optional_node_child("CompositionMetadataAsset")) {
1191 if (auto extension_list = metadata->optional_node_child("ExtensionMetadataList")) {
1193 for (auto extension: extension_list->node_children("ExtensionMetadata")) {
1194 if (extension->optional_string_attribute("scope").get_value_or("") != "http://isdcf.com/ns/cplmd/app") {
1198 if (auto name = extension->optional_node_child("Name")) {
1199 if (name->content() != "Application") {
1200 malformed = "<Name> should be 'Application'";
1203 if (auto property_list = extension->optional_node_child("PropertyList")) {
1204 if (auto property = property_list->optional_node_child("Property")) {
1205 if (auto name = property->optional_node_child("Name")) {
1206 if (name->content() != "DCP Constraints Profile") {
1207 malformed = "<Name> property should be 'DCP Constraints Profile'";
1210 if (auto value = property->optional_node_child("Value")) {
1211 if (value->content() != "SMPTE-RDD-52:2020-Bv2.1") {
1212 malformed = "<Value> property should be 'SMPTE-RDD-52:2020-Bv2.1'";
1227 notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_EXTENSION_METADATA, cpl->id(), cpl->file().get()});
1228 } else if (!malformed.empty()) {
1229 notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_EXTENSION_METADATA, malformed, cpl->file().get()});
1235 pkl_has_encrypted_assets(shared_ptr<const DCP> dcp, shared_ptr<const PKL> pkl)
1237 vector<string> encrypted;
1238 for (auto i: dcp->cpls()) {
1239 for (auto j: i->reel_file_assets()) {
1240 if (j->asset_ref().resolved()) {
1241 auto mxf = dynamic_pointer_cast<MXF>(j->asset_ref().asset());
1242 if (mxf && mxf->encrypted()) {
1243 encrypted.push_back(j->asset_ref().id());
1249 for (auto i: pkl->asset_list()) {
1250 if (find(encrypted.begin(), encrypted.end(), i->id()) != encrypted.end()) {
1262 shared_ptr<const DCP> dcp,
1263 shared_ptr<const CPL> cpl,
1264 shared_ptr<const Reel> reel,
1265 optional<dcp::Size> main_picture_active_area,
1266 function<void (string, optional<boost::filesystem::path>)> stage,
1267 boost::filesystem::path xsd_dtd_directory,
1268 function<void (float)> progress,
1269 vector<VerificationNote>& notes,
1271 bool* have_main_subtitle,
1272 bool* have_no_main_subtitle,
1273 size_t* most_closed_captions,
1274 size_t* fewest_closed_captions,
1275 map<Marker, Time>* markers_seen
1278 for (auto i: reel->assets()) {
1279 if (i->duration() && (i->duration().get() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
1280 notes.push_back({VerificationNote::Type::ERROR, VerificationNote::Code::INVALID_DURATION, i->id()});
1282 if ((i->intrinsic_duration() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
1283 notes.push_back({VerificationNote::Type::ERROR, VerificationNote::Code::INVALID_INTRINSIC_DURATION, i->id()});
1285 auto file_asset = dynamic_pointer_cast<ReelFileAsset>(i);
1286 if (i->encryptable() && !file_asset->hash()) {
1287 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_HASH, i->id()});
1291 if (dcp->standard() == Standard::SMPTE) {
1292 boost::optional<int64_t> duration;
1293 for (auto i: reel->assets()) {
1295 duration = i->actual_duration();
1296 } else if (*duration != i->actual_duration()) {
1297 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISMATCHED_ASSET_DURATION});
1303 if (reel->main_picture()) {
1304 /* Check reel stuff */
1305 auto const frame_rate = reel->main_picture()->frame_rate();
1306 if (frame_rate.denominator != 1 ||
1307 (frame_rate.numerator != 24 &&
1308 frame_rate.numerator != 25 &&
1309 frame_rate.numerator != 30 &&
1310 frame_rate.numerator != 48 &&
1311 frame_rate.numerator != 50 &&
1312 frame_rate.numerator != 60 &&
1313 frame_rate.numerator != 96)) {
1315 VerificationNote::Type::ERROR,
1316 VerificationNote::Code::INVALID_PICTURE_FRAME_RATE,
1317 String::compose("%1/%2", frame_rate.numerator, frame_rate.denominator)
1321 if (reel->main_picture()->asset_ref().resolved()) {
1322 verify_main_picture_asset(dcp, reel->main_picture(), stage, progress, notes);
1323 auto const asset_size = reel->main_picture()->asset()->size();
1324 if (main_picture_active_area) {
1325 if (main_picture_active_area->width > asset_size.width) {
1327 VerificationNote::Type::ERROR,
1328 VerificationNote::Code::INVALID_MAIN_PICTURE_ACTIVE_AREA,
1329 String::compose("width %1 is bigger than the asset width %2", main_picture_active_area->width, asset_size.width),
1333 if (main_picture_active_area->height > asset_size.height) {
1335 VerificationNote::Type::ERROR,
1336 VerificationNote::Code::INVALID_MAIN_PICTURE_ACTIVE_AREA,
1337 String::compose("height %1 is bigger than the asset height %2", main_picture_active_area->height, asset_size.height),
1345 if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
1346 verify_main_sound_asset(dcp, reel->main_sound(), stage, progress, notes);
1349 if (reel->main_subtitle()) {
1350 verify_main_subtitle_reel(reel->main_subtitle(), notes);
1351 if (reel->main_subtitle()->asset_ref().resolved()) {
1352 verify_subtitle_asset(reel->main_subtitle()->asset(), reel->main_subtitle()->duration(), stage, xsd_dtd_directory, notes, state);
1354 *have_main_subtitle = true;
1356 *have_no_main_subtitle = true;
1359 for (auto i: reel->closed_captions()) {
1360 verify_closed_caption_reel(i, notes);
1361 if (i->asset_ref().resolved()) {
1362 verify_closed_caption_asset(i->asset(), i->duration(), stage, xsd_dtd_directory, notes);
1366 if (reel->main_markers()) {
1367 for (auto const& i: reel->main_markers()->get()) {
1368 markers_seen->insert(i);
1370 if (reel->main_markers()->entry_point()) {
1371 notes.push_back({VerificationNote::Type::ERROR, VerificationNote::Code::UNEXPECTED_ENTRY_POINT});
1373 if (reel->main_markers()->duration()) {
1374 notes.push_back({VerificationNote::Type::ERROR, VerificationNote::Code::UNEXPECTED_DURATION});
1378 *fewest_closed_captions = std::min(*fewest_closed_captions, reel->closed_captions().size());
1379 *most_closed_captions = std::max(*most_closed_captions, reel->closed_captions().size());
1387 shared_ptr<const DCP> dcp,
1388 shared_ptr<const CPL> cpl,
1389 function<void (string, optional<boost::filesystem::path>)> stage,
1390 boost::filesystem::path xsd_dtd_directory,
1391 function<void (float)> progress,
1392 vector<VerificationNote>& notes,
1396 stage("Checking CPL", cpl->file());
1397 validate_xml(cpl->file().get(), xsd_dtd_directory, notes);
1399 if (cpl->any_encrypted() && !cpl->all_encrypted()) {
1400 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::PARTIALLY_ENCRYPTED});
1403 for (auto const& i: cpl->additional_subtitle_languages()) {
1404 verify_language_tag(i, notes);
1407 if (!cpl->content_kind().scope() || *cpl->content_kind().scope() == "http://www.smpte-ra.org/schemas/429-7/2006/CPL#standard-content") {
1408 /* This is a content kind from http://www.smpte-ra.org/schemas/429-7/2006/CPL#standard-content; make sure it's one
1409 * of the approved ones.
1411 auto all = ContentKind::all();
1412 auto name = cpl->content_kind().name();
1413 transform(name.begin(), name.end(), name.begin(), ::tolower);
1414 auto iter = std::find_if(all.begin(), all.end(), [name](ContentKind const& k) { return !k.scope() && k.name() == name; });
1415 if (iter == all.end()) {
1416 notes.push_back({VerificationNote::Type::ERROR, VerificationNote::Code::INVALID_CONTENT_KIND, cpl->content_kind().name()});
1420 if (cpl->release_territory()) {
1421 if (!cpl->release_territory_scope() || cpl->release_territory_scope().get() != "http://www.smpte-ra.org/schemas/429-16/2014/CPL-Metadata#scope/release-territory/UNM49") {
1422 auto terr = cpl->release_territory().get();
1423 /* Must be a valid region tag, or "001" */
1425 LanguageTag::RegionSubtag test(terr);
1427 if (terr != "001") {
1428 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_LANGUAGE, terr});
1434 if (dcp->standard() == Standard::SMPTE) {
1435 if (!cpl->annotation_text()) {
1436 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_CPL_ANNOTATION_TEXT, cpl->id(), cpl->file().get()});
1437 } else if (cpl->annotation_text().get() != cpl->content_title_text()) {
1438 notes.push_back({VerificationNote::Type::WARNING, VerificationNote::Code::MISMATCHED_CPL_ANNOTATION_TEXT, cpl->id(), cpl->file().get()});
1442 for (auto i: dcp->pkls()) {
1443 /* Check that the CPL's hash corresponds to the PKL */
1444 optional<string> h = i->hash(cpl->id());
1445 if (h && make_digest(ArrayData(*cpl->file())) != *h) {
1446 notes.push_back({VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_CPL_HASHES, cpl->id(), cpl->file().get()});
1449 /* Check that any PKL with a single CPL has its AnnotationText the same as the CPL's ContentTitleText */
1450 optional<string> required_annotation_text;
1451 for (auto j: i->asset_list()) {
1452 /* See if this is a CPL */
1453 for (auto k: dcp->cpls()) {
1454 if (j->id() == k->id()) {
1455 if (!required_annotation_text) {
1456 /* First CPL we have found; this is the required AnnotationText unless we find another */
1457 required_annotation_text = cpl->content_title_text();
1459 /* There's more than one CPL so we don't care what the PKL's AnnotationText is */
1460 required_annotation_text = boost::none;
1466 if (required_annotation_text && i->annotation_text() != required_annotation_text) {
1467 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISMATCHED_PKL_ANNOTATION_TEXT_WITH_CPL, i->id(), i->file().get()});
1471 /* set to true if any reel has a MainSubtitle */
1472 auto have_main_subtitle = false;
1473 /* set to true if any reel has no MainSubtitle */
1474 auto have_no_main_subtitle = false;
1475 /* fewest number of closed caption assets seen in a reel */
1476 size_t fewest_closed_captions = SIZE_MAX;
1477 /* most number of closed caption assets seen in a reel */
1478 size_t most_closed_captions = 0;
1479 map<Marker, Time> markers_seen;
1481 auto const main_picture_active_area = cpl->main_picture_active_area();
1482 if (main_picture_active_area && (main_picture_active_area->width % 2)) {
1484 VerificationNote::Type::ERROR,
1485 VerificationNote::Code::INVALID_MAIN_PICTURE_ACTIVE_AREA,
1486 String::compose("width %1 is not a multiple of 2", main_picture_active_area->width),
1490 if (main_picture_active_area && (main_picture_active_area->height % 2)) {
1492 VerificationNote::Type::ERROR,
1493 VerificationNote::Code::INVALID_MAIN_PICTURE_ACTIVE_AREA,
1494 String::compose("height %1 is not a multiple of 2", main_picture_active_area->height),
1499 for (auto reel: cpl->reels()) {
1500 stage("Checking reel", optional<boost::filesystem::path>());
1505 main_picture_active_area,
1511 &have_main_subtitle,
1512 &have_no_main_subtitle,
1513 &most_closed_captions,
1514 &fewest_closed_captions,
1519 verify_text_details(cpl->reels(), notes);
1521 if (dcp->standard() == Standard::SMPTE) {
1523 if (have_main_subtitle && have_no_main_subtitle) {
1524 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_MAIN_SUBTITLE_FROM_SOME_REELS});
1527 if (fewest_closed_captions != most_closed_captions) {
1528 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISMATCHED_CLOSED_CAPTION_ASSET_COUNTS});
1531 if (cpl->content_kind() == ContentKind::FEATURE) {
1532 if (markers_seen.find(Marker::FFEC) == markers_seen.end()) {
1533 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_FFEC_IN_FEATURE});
1535 if (markers_seen.find(Marker::FFMC) == markers_seen.end()) {
1536 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_FFMC_IN_FEATURE});
1540 auto ffoc = markers_seen.find(Marker::FFOC);
1541 if (ffoc == markers_seen.end()) {
1542 notes.push_back({VerificationNote::Type::WARNING, VerificationNote::Code::MISSING_FFOC});
1543 } else if (ffoc->second.e != 1) {
1544 notes.push_back({VerificationNote::Type::WARNING, VerificationNote::Code::INCORRECT_FFOC, raw_convert<string>(ffoc->second.e)});
1547 auto lfoc = markers_seen.find(Marker::LFOC);
1548 if (lfoc == markers_seen.end()) {
1549 notes.push_back({VerificationNote::Type::WARNING, VerificationNote::Code::MISSING_LFOC});
1551 auto lfoc_time = lfoc->second.as_editable_units_ceil(lfoc->second.tcr);
1552 if (lfoc_time != (cpl->reels().back()->duration() - 1)) {
1553 notes.push_back({VerificationNote::Type::WARNING, VerificationNote::Code::INCORRECT_LFOC, raw_convert<string>(lfoc_time)});
1557 LinesCharactersResult result;
1558 for (auto reel: cpl->reels()) {
1559 if (reel->main_subtitle() && reel->main_subtitle()->asset()) {
1560 verify_text_lines_and_characters(reel->main_subtitle()->asset(), 52, 79, &result);
1564 if (result.line_count_exceeded) {
1565 notes.push_back({VerificationNote::Type::WARNING, VerificationNote::Code::INVALID_SUBTITLE_LINE_COUNT});
1567 if (result.error_length_exceeded) {
1568 notes.push_back({VerificationNote::Type::WARNING, VerificationNote::Code::INVALID_SUBTITLE_LINE_LENGTH});
1569 } else if (result.warning_length_exceeded) {
1570 notes.push_back({VerificationNote::Type::WARNING, VerificationNote::Code::NEARLY_INVALID_SUBTITLE_LINE_LENGTH});
1573 result = LinesCharactersResult();
1574 for (auto reel: cpl->reels()) {
1575 for (auto i: reel->closed_captions()) {
1577 verify_text_lines_and_characters(i->asset(), 32, 32, &result);
1582 if (result.line_count_exceeded) {
1583 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_CLOSED_CAPTION_LINE_COUNT});
1585 if (result.error_length_exceeded) {
1586 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_CLOSED_CAPTION_LINE_LENGTH});
1589 if (!cpl->read_composition_metadata()) {
1590 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_CPL_METADATA, cpl->id(), cpl->file().get()});
1591 } else if (!cpl->version_number()) {
1592 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::MISSING_CPL_METADATA_VERSION_NUMBER, cpl->id(), cpl->file().get()});
1595 verify_extension_metadata(cpl, notes);
1597 if (cpl->any_encrypted()) {
1598 cxml::Document doc("CompositionPlaylist");
1599 DCP_ASSERT(cpl->file());
1600 doc.read_file(cpl->file().get());
1601 if (!doc.optional_node_child("Signature")) {
1602 notes.push_back({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::UNSIGNED_CPL_WITH_ENCRYPTED_CONTENT, cpl->id(), cpl->file().get()});
1609 vector<VerificationNote>
1611 vector<boost::filesystem::path> directories,
1612 function<void (string, optional<boost::filesystem::path>)> stage,
1613 function<void (float)> progress,
1614 optional<boost::filesystem::path> xsd_dtd_directory
1617 if (!xsd_dtd_directory) {
1618 xsd_dtd_directory = resources_directory() / "xsd";
1620 *xsd_dtd_directory = boost::filesystem::canonical (*xsd_dtd_directory);
1622 vector<VerificationNote> notes;
1625 vector<shared_ptr<DCP>> dcps;
1626 for (auto i: directories) {
1627 dcps.push_back (make_shared<DCP>(i));
1630 for (auto dcp: dcps) {
1631 stage ("Checking DCP", dcp->directory());
1632 bool carry_on = true;
1634 dcp->read (¬es, true);
1635 } catch (MissingAssetmapError& e) {
1636 notes.push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::FAILED_READ, string(e.what())});
1638 } catch (ReadError& e) {
1639 notes.push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::FAILED_READ, string(e.what())});
1640 } catch (XMLError& e) {
1641 notes.push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::FAILED_READ, string(e.what())});
1642 } catch (MXFFileError& e) {
1643 notes.push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::FAILED_READ, string(e.what())});
1644 } catch (cxml::Error& e) {
1645 notes.push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::FAILED_READ, string(e.what())});
1652 if (dcp->standard() != Standard::SMPTE) {
1653 notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::INVALID_STANDARD});
1656 for (auto cpl: dcp->cpls()) {
1668 for (auto pkl: dcp->pkls()) {
1669 stage ("Checking PKL", pkl->file());
1670 validate_xml (pkl->file().get(), *xsd_dtd_directory, notes);
1671 if (pkl_has_encrypted_assets(dcp, pkl)) {
1672 cxml::Document doc ("PackingList");
1673 doc.read_file (pkl->file().get());
1674 if (!doc.optional_node_child("Signature")) {
1675 notes.push_back ({VerificationNote::Type::BV21_ERROR, VerificationNote::Code::UNSIGNED_PKL_WITH_ENCRYPTED_CONTENT, pkl->id(), pkl->file().get()});
1680 if (dcp->asset_map_path()) {
1681 stage ("Checking ASSETMAP", dcp->asset_map_path().get());
1682 validate_xml (dcp->asset_map_path().get(), *xsd_dtd_directory, notes);
1684 notes.push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::MISSING_ASSETMAP});
1693 dcp::note_to_string (VerificationNote note)
1695 /** These strings should say what is wrong, incorporating any extra details (ID, filenames etc.).
1697 * e.g. "ClosedCaption asset has no <EntryPoint> tag.",
1698 * not "ClosedCaption assets must have an <EntryPoint> tag."
1700 * It's OK to use XML tag names where they are clear.
1701 * If both ID and filename are available, use only the ID.
1702 * End messages with a full stop.
1703 * Messages should not mention whether or not their errors are a part of Bv2.1.
1705 switch (note.code()) {
1706 case VerificationNote::Code::FAILED_READ:
1707 return *note.note();
1708 case VerificationNote::Code::MISMATCHED_CPL_HASHES:
1709 return String::compose("The hash of the CPL %1 in the PKL does not agree with the CPL file.", note.note().get());
1710 case VerificationNote::Code::INVALID_PICTURE_FRAME_RATE:
1711 return String::compose("The picture in a reel has an invalid frame rate %1.", note.note().get());
1712 case VerificationNote::Code::INCORRECT_PICTURE_HASH:
1713 return String::compose("The hash of the picture asset %1 does not agree with the PKL file.", note.file()->filename());
1714 case VerificationNote::Code::MISMATCHED_PICTURE_HASHES:
1715 return String::compose("The PKL and CPL hashes differ for the picture asset %1.", note.file()->filename());
1716 case VerificationNote::Code::INCORRECT_SOUND_HASH:
1717 return String::compose("The hash of the sound asset %1 does not agree with the PKL file.", note.file()->filename());
1718 case VerificationNote::Code::MISMATCHED_SOUND_HASHES:
1719 return String::compose("The PKL and CPL hashes differ for the sound asset %1.", note.file()->filename());
1720 case VerificationNote::Code::EMPTY_ASSET_PATH:
1721 return "The asset map contains an empty asset path.";
1722 case VerificationNote::Code::MISSING_ASSET:
1723 return String::compose("The file %1 for an asset in the asset map cannot be found.", note.file()->filename());
1724 case VerificationNote::Code::MISMATCHED_STANDARD:
1725 return "The DCP contains both SMPTE and Interop parts.";
1726 case VerificationNote::Code::INVALID_XML:
1727 return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get());
1728 case VerificationNote::Code::MISSING_ASSETMAP:
1729 return "No ASSETMAP or ASSETMAP.xml was found.";
1730 case VerificationNote::Code::INVALID_INTRINSIC_DURATION:
1731 return String::compose("The intrinsic duration of the asset %1 is less than 1 second.", note.note().get());
1732 case VerificationNote::Code::INVALID_DURATION:
1733 return String::compose("The duration of the asset %1 is less than 1 second.", note.note().get());
1734 case VerificationNote::Code::INVALID_PICTURE_FRAME_SIZE_IN_BYTES:
1735 return String::compose("The instantaneous bit rate of the picture asset %1 is larger than the limit of 250Mbit/s in at least one place.", note.file()->filename());
1736 case VerificationNote::Code::NEARLY_INVALID_PICTURE_FRAME_SIZE_IN_BYTES:
1737 return String::compose("The instantaneous bit rate of the picture asset %1 is close to the limit of 250Mbit/s in at least one place.", note.file()->filename());
1738 case VerificationNote::Code::EXTERNAL_ASSET:
1739 return String::compose("The asset %1 that this DCP refers to is not included in the DCP. It may be a VF.", note.note().get());
1740 case VerificationNote::Code::THREED_ASSET_MARKED_AS_TWOD:
1741 return String::compose("The asset %1 is 3D but its MXF is marked as 2D.", note.file()->filename());
1742 case VerificationNote::Code::INVALID_STANDARD:
1743 return "This DCP does not use the SMPTE standard.";
1744 case VerificationNote::Code::INVALID_LANGUAGE:
1745 return String::compose("The DCP specifies a language '%1' which does not conform to the RFC 5646 standard.", note.note().get());
1746 case VerificationNote::Code::INVALID_PICTURE_SIZE_IN_PIXELS:
1747 return String::compose("The size %1 of picture asset %2 is not allowed.", note.note().get(), note.file()->filename());
1748 case VerificationNote::Code::INVALID_PICTURE_FRAME_RATE_FOR_2K:
1749 return String::compose("The frame rate %1 of picture asset %2 is not allowed for 2K DCPs.", note.note().get(), note.file()->filename());
1750 case VerificationNote::Code::INVALID_PICTURE_FRAME_RATE_FOR_4K:
1751 return String::compose("The frame rate %1 of picture asset %2 is not allowed for 4K DCPs.", note.note().get(), note.file()->filename());
1752 case VerificationNote::Code::INVALID_PICTURE_ASSET_RESOLUTION_FOR_3D:
1753 return "3D 4K DCPs are not allowed.";
1754 case VerificationNote::Code::INVALID_CLOSED_CAPTION_XML_SIZE_IN_BYTES:
1755 return String::compose("The size %1 of the closed caption asset %2 is larger than the 256KB maximum.", note.note().get(), note.file()->filename());
1756 case VerificationNote::Code::INVALID_TIMED_TEXT_SIZE_IN_BYTES:
1757 return String::compose("The size %1 of the timed text asset %2 is larger than the 115MB maximum.", note.note().get(), note.file()->filename());
1758 case VerificationNote::Code::INVALID_TIMED_TEXT_FONT_SIZE_IN_BYTES:
1759 return String::compose("The size %1 of the fonts in timed text asset %2 is larger than the 10MB maximum.", note.note().get(), note.file()->filename());
1760 case VerificationNote::Code::MISSING_SUBTITLE_LANGUAGE:
1761 return String::compose("The XML for the SMPTE subtitle asset %1 has no <Language> tag.", note.file()->filename());
1762 case VerificationNote::Code::MISMATCHED_SUBTITLE_LANGUAGES:
1763 return "Some subtitle assets have different <Language> tags than others";
1764 case VerificationNote::Code::MISSING_SUBTITLE_START_TIME:
1765 return String::compose("The XML for the SMPTE subtitle asset %1 has no <StartTime> tag.", note.file()->filename());
1766 case VerificationNote::Code::INVALID_SUBTITLE_START_TIME:
1767 return String::compose("The XML for a SMPTE subtitle asset %1 has a non-zero <StartTime> tag.", note.file()->filename());
1768 case VerificationNote::Code::INVALID_SUBTITLE_FIRST_TEXT_TIME:
1769 return "The first subtitle or closed caption is less than 4 seconds from the start of the DCP.";
1770 case VerificationNote::Code::INVALID_SUBTITLE_DURATION:
1771 return "At least one subtitle lasts less than 15 frames.";
1772 case VerificationNote::Code::INVALID_SUBTITLE_SPACING:
1773 return "At least one pair of subtitles is separated by less than 2 frames.";
1774 case VerificationNote::Code::SUBTITLE_OVERLAPS_REEL_BOUNDARY:
1775 return "At least one subtitle extends outside of its reel.";
1776 case VerificationNote::Code::INVALID_SUBTITLE_LINE_COUNT:
1777 return "There are more than 3 subtitle lines in at least one place in the DCP.";
1778 case VerificationNote::Code::NEARLY_INVALID_SUBTITLE_LINE_LENGTH:
1779 return "There are more than 52 characters in at least one subtitle line.";
1780 case VerificationNote::Code::INVALID_SUBTITLE_LINE_LENGTH:
1781 return "There are more than 79 characters in at least one subtitle line.";
1782 case VerificationNote::Code::INVALID_CLOSED_CAPTION_LINE_COUNT:
1783 return "There are more than 3 closed caption lines in at least one place.";
1784 case VerificationNote::Code::INVALID_CLOSED_CAPTION_LINE_LENGTH:
1785 return "There are more than 32 characters in at least one closed caption line.";
1786 case VerificationNote::Code::INVALID_SOUND_FRAME_RATE:
1787 return String::compose("The sound asset %1 has a sampling rate of %2", note.file()->filename(), note.note().get());
1788 case VerificationNote::Code::MISSING_CPL_ANNOTATION_TEXT:
1789 return String::compose("The CPL %1 has no <AnnotationText> tag.", note.note().get());
1790 case VerificationNote::Code::MISMATCHED_CPL_ANNOTATION_TEXT:
1791 return String::compose("The CPL %1 has an <AnnotationText> which differs from its <ContentTitleText>.", note.note().get());
1792 case VerificationNote::Code::MISMATCHED_ASSET_DURATION:
1793 return "All assets in a reel do not have the same duration.";
1794 case VerificationNote::Code::MISSING_MAIN_SUBTITLE_FROM_SOME_REELS:
1795 return "At least one reel contains a subtitle asset, but some reel(s) do not.";
1796 case VerificationNote::Code::MISMATCHED_CLOSED_CAPTION_ASSET_COUNTS:
1797 return "At least one reel has closed captions, but reels have different numbers of closed caption assets.";
1798 case VerificationNote::Code::MISSING_SUBTITLE_ENTRY_POINT:
1799 return String::compose("The subtitle asset %1 has no <EntryPoint> tag.", note.note().get());
1800 case VerificationNote::Code::INCORRECT_SUBTITLE_ENTRY_POINT:
1801 return String::compose("The subtitle asset %1 has an <EntryPoint> other than 0.", note.note().get());
1802 case VerificationNote::Code::MISSING_CLOSED_CAPTION_ENTRY_POINT:
1803 return String::compose("The closed caption asset %1 has no <EntryPoint> tag.", note.note().get());
1804 case VerificationNote::Code::INCORRECT_CLOSED_CAPTION_ENTRY_POINT:
1805 return String::compose("The closed caption asset %1 has an <EntryPoint> other than 0.", note.note().get());
1806 case VerificationNote::Code::MISSING_HASH:
1807 return String::compose("The asset %1 has no <Hash> tag in the CPL.", note.note().get());
1808 case VerificationNote::Code::MISSING_FFEC_IN_FEATURE:
1809 return "The DCP is marked as a Feature but there is no FFEC (first frame of end credits) marker.";
1810 case VerificationNote::Code::MISSING_FFMC_IN_FEATURE:
1811 return "The DCP is marked as a Feature but there is no FFMC (first frame of moving credits) marker.";
1812 case VerificationNote::Code::MISSING_FFOC:
1813 return "There should be a FFOC (first frame of content) marker.";
1814 case VerificationNote::Code::MISSING_LFOC:
1815 return "There should be a LFOC (last frame of content) marker.";
1816 case VerificationNote::Code::INCORRECT_FFOC:
1817 return String::compose("The FFOC marker is %1 instead of 1", note.note().get());
1818 case VerificationNote::Code::INCORRECT_LFOC:
1819 return String::compose("The LFOC marker is %1 instead of 1 less than the duration of the last reel.", note.note().get());
1820 case VerificationNote::Code::MISSING_CPL_METADATA:
1821 return String::compose("The CPL %1 has no <CompositionMetadataAsset> tag.", note.note().get());
1822 case VerificationNote::Code::MISSING_CPL_METADATA_VERSION_NUMBER:
1823 return String::compose("The CPL %1 has no <VersionNumber> in its <CompositionMetadataAsset>.", note.note().get());
1824 case VerificationNote::Code::MISSING_EXTENSION_METADATA:
1825 return String::compose("The CPL %1 has no <ExtensionMetadata> in its <CompositionMetadataAsset>.", note.note().get());
1826 case VerificationNote::Code::INVALID_EXTENSION_METADATA:
1827 return String::compose("The CPL %1 has a malformed <ExtensionMetadata> (%2).", note.file()->filename(), note.note().get());
1828 case VerificationNote::Code::UNSIGNED_CPL_WITH_ENCRYPTED_CONTENT:
1829 return String::compose("The CPL %1, which has encrypted content, is not signed.", note.note().get());
1830 case VerificationNote::Code::UNSIGNED_PKL_WITH_ENCRYPTED_CONTENT:
1831 return String::compose("The PKL %1, which has encrypted content, is not signed.", note.note().get());
1832 case VerificationNote::Code::MISMATCHED_PKL_ANNOTATION_TEXT_WITH_CPL:
1833 return String::compose("The PKL %1 has only one CPL but its <AnnotationText> does not match the CPL's <ContentTitleText>.", note.note().get());
1834 case VerificationNote::Code::PARTIALLY_ENCRYPTED:
1835 return "Some assets are encrypted but some are not.";
1836 case VerificationNote::Code::INVALID_JPEG2000_CODESTREAM:
1837 return String::compose("The JPEG2000 codestream for at least one frame is invalid (%1).", note.note().get());
1838 case VerificationNote::Code::INVALID_JPEG2000_GUARD_BITS_FOR_2K:
1839 return String::compose("The JPEG2000 codestream uses %1 guard bits in a 2K image instead of 1.", note.note().get());
1840 case VerificationNote::Code::INVALID_JPEG2000_GUARD_BITS_FOR_4K:
1841 return String::compose("The JPEG2000 codestream uses %1 guard bits in a 4K image instead of 2.", note.note().get());
1842 case VerificationNote::Code::INVALID_JPEG2000_TILE_SIZE:
1843 return "The JPEG2000 tile size is not the same as the image size.";
1844 case VerificationNote::Code::INVALID_JPEG2000_CODE_BLOCK_WIDTH:
1845 return String::compose("The JPEG2000 codestream uses a code block width of %1 instead of 32.", note.note().get());
1846 case VerificationNote::Code::INVALID_JPEG2000_CODE_BLOCK_HEIGHT:
1847 return String::compose("The JPEG2000 codestream uses a code block height of %1 instead of 32.", note.note().get());
1848 case VerificationNote::Code::INCORRECT_JPEG2000_POC_MARKER_COUNT_FOR_2K:
1849 return String::compose("%1 POC markers found in 2K JPEG2000 codestream instead of 0.", note.note().get());
1850 case VerificationNote::Code::INCORRECT_JPEG2000_POC_MARKER_COUNT_FOR_4K:
1851 return String::compose("%1 POC markers found in 4K JPEG2000 codestream instead of 1.", note.note().get());
1852 case VerificationNote::Code::INCORRECT_JPEG2000_POC_MARKER:
1853 return String::compose("Incorrect POC marker content found (%1).", note.note().get());
1854 case VerificationNote::Code::INVALID_JPEG2000_POC_MARKER_LOCATION:
1855 return "POC marker found outside main header.";
1856 case VerificationNote::Code::INVALID_JPEG2000_TILE_PARTS_FOR_2K:
1857 return String::compose("The JPEG2000 codestream has %1 tile parts in a 2K image instead of 3.", note.note().get());
1858 case VerificationNote::Code::INVALID_JPEG2000_TILE_PARTS_FOR_4K:
1859 return String::compose("The JPEG2000 codestream has %1 tile parts in a 4K image instead of 6.", note.note().get());
1860 case VerificationNote::Code::MISSING_JPEG200_TLM_MARKER:
1861 return "No TLM marker was found in a JPEG2000 codestream.";
1862 case VerificationNote::Code::MISMATCHED_TIMED_TEXT_RESOURCE_ID:
1863 return "The Resource ID in a timed text MXF did not match the ID of the contained XML.";
1864 case VerificationNote::Code::INCORRECT_TIMED_TEXT_ASSET_ID:
1865 return "The Asset ID in a timed text MXF is the same as the Resource ID or that of the contained XML.";
1866 case VerificationNote::Code::MISMATCHED_TIMED_TEXT_DURATION:
1868 vector<string> parts;
1869 boost::split (parts, note.note().get(), boost::is_any_of(" "));
1870 DCP_ASSERT (parts.size() == 2);
1871 return String::compose("The reel duration of some timed text (%1) is not the same as the ContainerDuration of its MXF (%2).", parts[0], parts[1]);
1873 case VerificationNote::Code::MISSED_CHECK_OF_ENCRYPTED:
1874 return "Some aspect of this DCP could not be checked because it is encrypted.";
1875 case VerificationNote::Code::EMPTY_TEXT:
1876 return "There is an empty <Text> node in a subtitle or closed caption.";
1877 case VerificationNote::Code::MISMATCHED_CLOSED_CAPTION_VALIGN:
1878 return "Some closed <Text> or <Image> nodes have different vertical alignments within a <Subtitle>.";
1879 case VerificationNote::Code::INCORRECT_CLOSED_CAPTION_ORDERING:
1880 return "Some closed captions are not listed in the order of their vertical position.";
1881 case VerificationNote::Code::UNEXPECTED_ENTRY_POINT:
1882 return "There is an <EntryPoint> node inside a <MainMarkers>.";
1883 case VerificationNote::Code::UNEXPECTED_DURATION:
1884 return "There is an <Duration> node inside a <MainMarkers>.";
1885 case VerificationNote::Code::INVALID_CONTENT_KIND:
1886 return String::compose("<ContentKind> has an invalid value %1.", note.note().get());
1887 case VerificationNote::Code::INVALID_MAIN_PICTURE_ACTIVE_AREA:
1888 return String::compose("<MainPictureActiveaArea> has an invalid value: %1", note.note().get());
1896 dcp::operator== (dcp::VerificationNote const& a, dcp::VerificationNote const& b)
1898 return a.type() == b.type() && a.code() == b.code() && a.note() == b.note() && a.file() == b.file() && a.line() == b.line();
1903 dcp::operator< (dcp::VerificationNote const& a, dcp::VerificationNote const& b)
1905 if (a.type() != b.type()) {
1906 return a.type() < b.type();
1909 if (a.code() != b.code()) {
1910 return a.code() < b.code();
1913 if (a.note() != b.note()) {
1914 return a.note().get_value_or("") < b.note().get_value_or("");
1917 if (a.file() != b.file()) {
1918 return a.file().get_value_or("") < b.file().get_value_or("");
1921 return a.line().get_value_or(0) < b.line().get_value_or(0);
1926 dcp::operator<< (std::ostream& s, dcp::VerificationNote const& note)
1928 s << note_to_string (note);
1930 s << " [" << note.note().get() << "]";
1933 s << " [" << note.file().get() << "]";
1936 s << " [" << note.line().get() << "]";