2 Copyright (C) 2018-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of libdcp.
6 libdcp is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 libdcp is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with libdcp. If not, see <http://www.gnu.org/licenses/>.
19 In addition, as a special exception, the copyright holders give
20 permission to link the code of portions of this program with the
21 OpenSSL library under certain conditions as described in each
22 individual source file, and distribute linked combinations
25 You must obey the GNU General Public License in all respects
26 for all of the code used other than OpenSSL. If you modify
27 file(s) with this exception, you may extend this exception to your
28 version of the file(s), but you are not obligated to do so. If you
29 do not wish to do so, delete this exception statement from your
30 version. If you delete this exception statement from all source
31 files in the program, then also delete it here.
38 #include "reel_closed_caption_asset.h"
39 #include "reel_picture_asset.h"
40 #include "reel_sound_asset.h"
41 #include "reel_subtitle_asset.h"
42 #include "interop_subtitle_asset.h"
43 #include "mono_picture_asset.h"
44 #include "mono_picture_frame.h"
45 #include "stereo_picture_asset.h"
46 #include "stereo_picture_frame.h"
47 #include "exceptions.h"
48 #include "compose.hpp"
49 #include "raw_convert.h"
50 #include "smpte_subtitle_asset.h"
51 #include <xercesc/util/PlatformUtils.hpp>
52 #include <xercesc/parsers/XercesDOMParser.hpp>
53 #include <xercesc/parsers/AbstractDOMParser.hpp>
54 #include <xercesc/sax/HandlerBase.hpp>
55 #include <xercesc/dom/DOMImplementation.hpp>
56 #include <xercesc/dom/DOMImplementationLS.hpp>
57 #include <xercesc/dom/DOMImplementationRegistry.hpp>
58 #include <xercesc/dom/DOMLSParser.hpp>
59 #include <xercesc/dom/DOMException.hpp>
60 #include <xercesc/dom/DOMDocument.hpp>
61 #include <xercesc/dom/DOMNodeList.hpp>
62 #include <xercesc/dom/DOMError.hpp>
63 #include <xercesc/dom/DOMLocator.hpp>
64 #include <xercesc/dom/DOMNamedNodeMap.hpp>
65 #include <xercesc/dom/DOMAttr.hpp>
66 #include <xercesc/dom/DOMErrorHandler.hpp>
67 #include <xercesc/framework/LocalFileInputSource.hpp>
68 #include <xercesc/framework/MemBufInputSource.hpp>
69 #include <boost/noncopyable.hpp>
70 #include <boost/algorithm/string.hpp>
81 using std::shared_ptr;
82 using std::make_shared;
83 using boost::optional;
84 using boost::function;
85 using std::dynamic_pointer_cast;
88 using namespace xercesc;
92 xml_ch_to_string (XMLCh const * a)
94 char* x = XMLString::transcode(a);
96 XMLString::release(&x);
100 class XMLValidationError
103 XMLValidationError (SAXParseException const & e)
104 : _message (xml_ch_to_string(e.getMessage()))
105 , _line (e.getLineNumber())
106 , _column (e.getColumnNumber())
107 , _public_id (e.getPublicId() ? xml_ch_to_string(e.getPublicId()) : "")
108 , _system_id (e.getSystemId() ? xml_ch_to_string(e.getSystemId()) : "")
113 string message () const {
117 uint64_t line () const {
121 uint64_t column () const {
125 string public_id () const {
129 string system_id () const {
142 class DCPErrorHandler : public ErrorHandler
145 void warning(const SAXParseException& e)
147 maybe_add (XMLValidationError(e));
150 void error(const SAXParseException& e)
152 maybe_add (XMLValidationError(e));
155 void fatalError(const SAXParseException& e)
157 maybe_add (XMLValidationError(e));
164 list<XMLValidationError> errors () const {
169 void maybe_add (XMLValidationError e)
171 /* XXX: nasty hack */
173 e.message().find("schema document") != string::npos &&
174 e.message().find("has different target namespace from the one specified in instance document") != string::npos
179 _errors.push_back (e);
182 list<XMLValidationError> _errors;
185 class StringToXMLCh : public boost::noncopyable
188 StringToXMLCh (string a)
190 _buffer = XMLString::transcode(a.c_str());
195 XMLString::release (&_buffer);
198 XMLCh const * get () const {
206 class LocalFileResolver : public EntityResolver
209 LocalFileResolver (boost::filesystem::path xsd_dtd_directory)
210 : _xsd_dtd_directory (xsd_dtd_directory)
212 /* XXX: I'm not clear on what things need to be in this list; some XSDs are apparently, magically
213 * found without being here.
215 add("http://www.w3.org/2001/XMLSchema.dtd", "XMLSchema.dtd");
216 add("http://www.w3.org/2001/03/xml.xsd", "xml.xsd");
217 add("http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd", "xmldsig-core-schema.xsd");
218 add("http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd", "Main-Stereo-Picture-CPL.xsd");
219 add("http://www.digicine.com/PROTO-ASDCP-CPL-20040511.xsd", "PROTO-ASDCP-CPL-20040511.xsd");
220 add("http://www.digicine.com/PROTO-ASDCP-PKL-20040311.xsd", "PROTO-ASDCP-PKL-20040311.xsd");
221 add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd");
222 add("http://www.digicine.com/PROTO-ASDCP-CC-CPL-20070926#", "PROTO-ASDCP-CC-CPL-20070926.xsd");
223 add("interop-subs", "DCSubtitle.v1.mattsson.xsd");
224 add("http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd", "SMPTE-428-7-2010-DCST.xsd");
225 add("http://www.smpte-ra.org/schemas/429-16/2014/CPL-Metadata", "SMPTE-429-16.xsd");
226 add("http://www.dolby.com/schemas/2012/AD", "Dolby-2012-AD.xsd");
227 add("http://www.smpte-ra.org/schemas/429-10/2008/Main-Stereo-Picture-CPL", "SMPTE-429-10-2008.xsd");
230 InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id)
235 auto system_id_str = xml_ch_to_string (system_id);
236 auto p = _xsd_dtd_directory;
237 if (_files.find(system_id_str) == _files.end()) {
240 p /= _files[system_id_str];
242 StringToXMLCh ch (p.string());
243 return new LocalFileInputSource(ch.get());
247 void add (string uri, string file)
252 std::map<string, string> _files;
253 boost::filesystem::path _xsd_dtd_directory;
258 parse (XercesDOMParser& parser, boost::filesystem::path xml)
260 parser.parse(xml.string().c_str());
265 parse (XercesDOMParser& parser, string xml)
267 xercesc::MemBufInputSource buf(reinterpret_cast<unsigned char const*>(xml.c_str()), xml.size(), "");
274 validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, vector<VerificationNote>& notes)
277 XMLPlatformUtils::Initialize ();
278 } catch (XMLException& e) {
279 throw MiscError ("Failed to initialise xerces library");
282 DCPErrorHandler error_handler;
284 /* All the xerces objects in this scope must be destroyed before XMLPlatformUtils::Terminate() is called */
286 XercesDOMParser parser;
287 parser.setValidationScheme(XercesDOMParser::Val_Always);
288 parser.setDoNamespaces(true);
289 parser.setDoSchema(true);
291 vector<string> schema;
292 schema.push_back("xml.xsd");
293 schema.push_back("xmldsig-core-schema.xsd");
294 schema.push_back("SMPTE-429-7-2006-CPL.xsd");
295 schema.push_back("SMPTE-429-8-2006-PKL.xsd");
296 schema.push_back("SMPTE-429-9-2007-AM.xsd");
297 schema.push_back("Main-Stereo-Picture-CPL.xsd");
298 schema.push_back("PROTO-ASDCP-CPL-20040511.xsd");
299 schema.push_back("PROTO-ASDCP-PKL-20040311.xsd");
300 schema.push_back("PROTO-ASDCP-AM-20040311.xsd");
301 schema.push_back("DCSubtitle.v1.mattsson.xsd");
302 schema.push_back("DCDMSubtitle-2010.xsd");
303 schema.push_back("PROTO-ASDCP-CC-CPL-20070926.xsd");
304 schema.push_back("SMPTE-429-16.xsd");
305 schema.push_back("Dolby-2012-AD.xsd");
306 schema.push_back("SMPTE-429-10-2008.xsd");
307 schema.push_back("xlink.xsd");
308 schema.push_back("SMPTE-335-2012.xsd");
309 schema.push_back("SMPTE-395-2014-13-1-aaf.xsd");
310 schema.push_back("isdcf-mca.xsd");
311 schema.push_back("SMPTE-429-12-2008.xsd");
313 /* XXX: I'm not especially clear what this is for, but it seems to be necessary.
314 * Schemas that are not mentioned in this list are not read, and the things
315 * they describe are not checked.
318 for (auto i: schema) {
319 locations += String::compose("%1 %1 ", i, i);
322 parser.setExternalSchemaLocation(locations.c_str());
323 parser.setValidationSchemaFullChecking(true);
324 parser.setErrorHandler(&error_handler);
326 LocalFileResolver resolver (xsd_dtd_directory);
327 parser.setEntityResolver(&resolver);
330 parser.resetDocumentPool();
332 } catch (XMLException& e) {
333 throw MiscError(xml_ch_to_string(e.getMessage()));
334 } catch (DOMException& e) {
335 throw MiscError(xml_ch_to_string(e.getMessage()));
337 throw MiscError("Unknown exception from xerces");
341 XMLPlatformUtils::Terminate ();
343 for (auto i: error_handler.errors()) {
346 VerificationNote::VERIFY_ERROR,
347 VerificationNote::XML_VALIDATION_ERROR,
349 boost::trim_copy(i.public_id() + " " + i.system_id()),
357 enum VerifyAssetResult {
358 VERIFY_ASSET_RESULT_GOOD,
359 VERIFY_ASSET_RESULT_CPL_PKL_DIFFER,
360 VERIFY_ASSET_RESULT_BAD
364 static VerifyAssetResult
365 verify_asset (shared_ptr<const DCP> dcp, shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
367 auto const actual_hash = reel_mxf->asset_ref()->hash(progress);
369 auto pkls = dcp->pkls();
370 /* We've read this DCP in so it must have at least one PKL */
371 DCP_ASSERT (!pkls.empty());
373 auto asset = reel_mxf->asset_ref().asset();
375 optional<string> pkl_hash;
377 pkl_hash = i->hash (reel_mxf->asset_ref()->id());
383 DCP_ASSERT (pkl_hash);
385 auto cpl_hash = reel_mxf->hash();
386 if (cpl_hash && *cpl_hash != *pkl_hash) {
387 return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER;
390 if (actual_hash != *pkl_hash) {
391 return VERIFY_ASSET_RESULT_BAD;
394 return VERIFY_ASSET_RESULT_GOOD;
399 verify_language_tag (string tag, vector<VerificationNote>& notes)
402 dcp::LanguageTag test (tag);
403 } catch (dcp::LanguageTagError &) {
404 notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::BAD_LANGUAGE, tag));
409 enum VerifyPictureAssetResult
411 VERIFY_PICTURE_ASSET_RESULT_GOOD,
412 VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE,
413 VERIFY_PICTURE_ASSET_RESULT_BAD,
418 biggest_frame_size (shared_ptr<const MonoPictureFrame> frame)
420 return frame->size ();
424 biggest_frame_size (shared_ptr<const StereoPictureFrame> frame)
426 return max(frame->left()->size(), frame->right()->size());
430 template <class A, class R, class F>
431 optional<VerifyPictureAssetResult>
432 verify_picture_asset_type (shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
434 auto asset = dynamic_pointer_cast<A>(reel_mxf->asset_ref().asset());
436 return optional<VerifyPictureAssetResult>();
439 int biggest_frame = 0;
440 auto reader = asset->start_read ();
441 auto const duration = asset->intrinsic_duration ();
442 for (int64_t i = 0; i < duration; ++i) {
443 shared_ptr<const F> frame = reader->get_frame (i);
444 biggest_frame = max(biggest_frame, biggest_frame_size(frame));
445 progress (float(i) / duration);
448 static const int max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float()));
449 static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float()));
450 if (biggest_frame > max_frame) {
451 return VERIFY_PICTURE_ASSET_RESULT_BAD;
452 } else if (biggest_frame > risky_frame) {
453 return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE;
456 return VERIFY_PICTURE_ASSET_RESULT_GOOD;
460 static VerifyPictureAssetResult
461 verify_picture_asset (shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
463 auto r = verify_picture_asset_type<MonoPictureAsset, MonoPictureAssetReader, MonoPictureFrame>(reel_mxf, progress);
465 r = verify_picture_asset_type<StereoPictureAsset, StereoPictureAssetReader, StereoPictureFrame>(reel_mxf, progress);
474 verify_main_picture_asset (
475 shared_ptr<const DCP> dcp,
476 shared_ptr<const ReelPictureAsset> reel_asset,
477 function<void (string, optional<boost::filesystem::path>)> stage,
478 function<void (float)> progress,
479 vector<VerificationNote>& notes
482 auto asset = reel_asset->asset();
483 auto const file = *asset->file();
484 stage ("Checking picture asset hash", file);
485 auto const r = verify_asset (dcp, reel_asset, progress);
487 case VERIFY_ASSET_RESULT_BAD:
490 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, file
494 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
497 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DIFFER, file
504 stage ("Checking picture frame sizes", asset->file());
505 auto const pr = verify_picture_asset (reel_asset, progress);
507 case VERIFY_PICTURE_ASSET_RESULT_BAD:
510 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_FRAME_TOO_LARGE_IN_BYTES, file
514 case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE:
517 VerificationNote::VERIFY_WARNING, VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE_IN_BYTES, file
525 /* Only flat/scope allowed by Bv2.1 */
527 asset->size() != dcp::Size(2048, 858) &&
528 asset->size() != dcp::Size(1998, 1080) &&
529 asset->size() != dcp::Size(4096, 1716) &&
530 asset->size() != dcp::Size(3996, 2160)) {
533 VerificationNote::VERIFY_BV21_ERROR,
534 VerificationNote::PICTURE_ASSET_INVALID_SIZE_IN_PIXELS,
535 String::compose("%1x%2", asset->size().width, asset->size().height),
541 /* Only 24, 25, 48fps allowed for 2K */
543 (asset->size() == dcp::Size(2048, 858) || asset->size() == dcp::Size(1998, 1080)) &&
544 (asset->edit_rate() != dcp::Fraction(24, 1) && asset->edit_rate() != dcp::Fraction(25, 1) && asset->edit_rate() != dcp::Fraction(48, 1))
548 VerificationNote::VERIFY_BV21_ERROR,
549 VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_2K,
550 String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator),
556 if (asset->size() == dcp::Size(4096, 1716) || asset->size() == dcp::Size(3996, 2160)) {
557 /* Only 24fps allowed for 4K */
558 if (asset->edit_rate() != dcp::Fraction(24, 1)) {
561 VerificationNote::VERIFY_BV21_ERROR,
562 VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_4K,
563 String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator),
569 /* Only 2D allowed for 4K */
570 if (dynamic_pointer_cast<const StereoPictureAsset>(asset)) {
573 VerificationNote::VERIFY_BV21_ERROR,
574 VerificationNote::PICTURE_ASSET_4K_3D,
586 verify_main_sound_asset (
587 shared_ptr<const DCP> dcp,
588 shared_ptr<const ReelSoundAsset> reel_asset,
589 function<void (string, optional<boost::filesystem::path>)> stage,
590 function<void (float)> progress,
591 vector<VerificationNote>& notes
594 auto asset = reel_asset->asset();
595 stage ("Checking sound asset hash", asset->file());
596 auto const r = verify_asset (dcp, reel_asset, progress);
598 case VERIFY_ASSET_RESULT_BAD:
601 VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *asset->file()
605 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
608 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DIFFER, *asset->file()
616 stage ("Checking sound asset metadata", asset->file());
618 verify_language_tag (asset->language(), notes);
619 if (asset->sampling_rate() != 48000) {
622 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::INVALID_SOUND_FRAME_RATE, *asset->file()
630 verify_main_subtitle_reel (shared_ptr<const ReelSubtitleAsset> reel_asset, vector<VerificationNote>& notes)
632 /* XXX: is Language compulsory? */
633 if (reel_asset->language()) {
634 verify_language_tag (*reel_asset->language(), notes);
637 if (!reel_asset->entry_point()) {
638 notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_ENTRY_POINT });
639 } else if (reel_asset->entry_point().get()) {
640 notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::SUBTITLE_ENTRY_POINT_NON_ZERO });
646 verify_closed_caption_reel (shared_ptr<const ReelClosedCaptionAsset> reel_asset, vector<VerificationNote>& notes)
648 /* XXX: is Language compulsory? */
649 if (reel_asset->language()) {
650 verify_language_tag (*reel_asset->language(), notes);
653 if (!reel_asset->entry_point()) {
654 notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_CLOSED_CAPTION_ENTRY_POINT });
655 } else if (reel_asset->entry_point().get()) {
656 notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::CLOSED_CAPTION_ENTRY_POINT_NON_ZERO });
663 boost::optional<string> subtitle_language;
669 verify_smpte_subtitle_asset (
670 shared_ptr<const dcp::SMPTESubtitleAsset> asset,
671 vector<VerificationNote>& notes,
675 if (asset->language()) {
676 auto const language = *asset->language();
677 verify_language_tag (language, notes);
678 if (!state.subtitle_language) {
679 state.subtitle_language = language;
680 } else if (state.subtitle_language != language) {
683 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::SUBTITLE_LANGUAGES_DIFFER, *asset->file()
690 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_LANGUAGE, *asset->file()
694 if (boost::filesystem::file_size(*asset->file()) > 115 * 1024 * 1024) {
697 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::TIMED_TEXT_ASSET_TOO_LARGE_IN_BYTES, *asset->file()
701 /* XXX: I'm not sure what Bv2.1_7.2.1 means when it says "the font resource shall not be larger than 10MB"
702 * but I'm hoping that checking for the total size of all fonts being <= 10MB will do.
704 auto fonts = asset->font_data ();
706 for (auto i: fonts) {
707 total_size += i.second.size();
709 if (total_size > 10 * 1024 * 1024) {
712 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::TIMED_TEXT_FONTS_TOO_LARGE_IN_BYTES, *asset->file()
717 if (!asset->start_time()) {
720 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_START_TIME, *asset->file())
722 } else if (asset->start_time() != dcp::Time()) {
725 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::SUBTITLE_START_TIME_NON_ZERO, *asset->file())
732 verify_subtitle_asset (
733 shared_ptr<const SubtitleAsset> asset,
734 function<void (string, optional<boost::filesystem::path>)> stage,
735 boost::filesystem::path xsd_dtd_directory,
736 vector<VerificationNote>& notes,
740 stage ("Checking subtitle XML", asset->file());
741 /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk
742 * gets passed through libdcp which may clean up and therefore hide errors.
744 validate_xml (asset->raw_xml(), xsd_dtd_directory, notes);
746 auto smpte = dynamic_pointer_cast<const SMPTESubtitleAsset>(asset);
748 verify_smpte_subtitle_asset (smpte, notes, state);
754 verify_closed_caption_asset (
755 shared_ptr<const SubtitleAsset> asset,
756 function<void (string, optional<boost::filesystem::path>)> stage,
757 boost::filesystem::path xsd_dtd_directory,
758 vector<VerificationNote>& notes,
762 verify_subtitle_asset (asset, stage, xsd_dtd_directory, notes, state);
764 if (asset->raw_xml().size() > 256 * 1024) {
767 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::CLOSED_CAPTION_XML_TOO_LARGE_IN_BYTES, *asset->file()
777 vector<shared_ptr<dcp::Reel>> reels,
778 optional<int> picture_frame_rate,
779 vector<VerificationNote>& notes,
780 std::function<bool (shared_ptr<dcp::Reel>)> check,
781 std::function<string (shared_ptr<dcp::Reel>)> xml,
782 std::function<int64_t (shared_ptr<dcp::Reel>)> duration
785 /* end of last subtitle (in editable units) */
786 optional<int64_t> last_out;
787 auto too_short = false;
788 auto too_close = false;
789 auto too_early = false;
790 /* current reel start time (in editable units) */
791 int64_t reel_offset = 0;
793 std::function<void (cxml::ConstNodePtr, int, int, bool)> parse;
794 parse = [&parse, &last_out, &too_short, &too_close, &too_early, &reel_offset](cxml::ConstNodePtr node, int tcr, int pfr, bool first_reel) {
795 if (node->name() == "Subtitle") {
796 dcp::Time in (node->string_attribute("TimeIn"), tcr);
797 dcp::Time out (node->string_attribute("TimeOut"), tcr);
798 if (first_reel && in < dcp::Time(0, 0, 4, 0, tcr)) {
801 auto length = out - in;
802 if (length.as_editable_units(pfr) < 15) {
806 /* XXX: this feels dubious - is it really what Bv2.1 means? */
807 auto distance = reel_offset + in.as_editable_units(pfr) - *last_out;
808 if (distance >= 0 && distance < 2) {
812 last_out = reel_offset + out.as_editable_units(pfr);
814 for (auto i: node->node_children()) {
815 parse(i, tcr, pfr, first_reel);
820 for (auto i = 0U; i < reels.size(); ++i) {
821 if (!check(reels[i])) {
825 /* We need to look at <Subtitle> instances in the XML being checked, so we can't use the subtitles
826 * read in by libdcp's parser.
829 auto doc = make_shared<cxml::Document>("SubtitleReel");
830 doc->read_string (xml(reels[i]));
831 auto const tcr = doc->number_child<int>("TimeCodeRate");
832 parse (doc, tcr, picture_frame_rate.get_value_or(24), i == 0);
833 reel_offset += duration(reels[i]);
839 VerificationNote::VERIFY_WARNING, VerificationNote::FIRST_TEXT_TOO_EARLY
847 VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_TOO_SHORT
855 VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_TOO_CLOSE
862 struct LinesCharactersResult
864 bool warning_length_exceeded = false;
865 bool error_length_exceeded = false;
866 bool line_count_exceeded = false;
872 check_text_lines_and_characters (
873 shared_ptr<SubtitleAsset> asset,
876 LinesCharactersResult* result
882 Event (dcp::Time time_, float position_, int characters_)
884 , position (position_)
885 , characters (characters_)
888 Event (dcp::Time time_, shared_ptr<Event> start_)
894 int position; //< position from 0 at top of screen to 100 at bottom
896 shared_ptr<Event> start;
899 vector<shared_ptr<Event>> events;
901 auto position = [](shared_ptr<const SubtitleString> sub) {
902 switch (sub->v_align()) {
904 return lrintf(sub->v_position() * 100);
906 return lrintf((0.5f + sub->v_position()) * 100);
908 return lrintf((1.0f - sub->v_position()) * 100);
914 for (auto j: asset->subtitles()) {
915 auto text = dynamic_pointer_cast<const SubtitleString>(j);
917 auto in = make_shared<Event>(text->in(), position(text), text->text().length());
918 events.push_back(in);
919 events.push_back(make_shared<Event>(text->out(), in));
923 std::sort(events.begin(), events.end(), [](shared_ptr<Event> const& a, shared_ptr<Event>const& b) {
924 return a->time < b->time;
927 map<int, int> current;
928 for (auto i: events) {
929 if (current.size() > 3) {
930 result->line_count_exceeded = true;
932 for (auto j: current) {
933 if (j.second >= warning_length) {
934 result->warning_length_exceeded = true;
936 if (j.second >= error_length) {
937 result->error_length_exceeded = true;
942 /* end of a subtitle */
943 DCP_ASSERT (current.find(i->start->position) != current.end());
944 if (current[i->start->position] == i->start->characters) {
945 current.erase(i->start->position);
947 current[i->start->position] -= i->start->characters;
950 /* start of a subtitle */
951 if (current.find(i->position) == current.end()) {
952 current[i->position] = i->characters;
954 current[i->position] += i->characters;
963 check_text_timing (vector<shared_ptr<dcp::Reel>> reels, vector<VerificationNote>& notes)
969 optional<int> picture_frame_rate;
970 if (reels[0]->main_picture()) {
971 picture_frame_rate = reels[0]->main_picture()->frame_rate().numerator;
974 if (reels[0]->main_subtitle()) {
975 check_text_timing (reels, picture_frame_rate, notes,
976 [](shared_ptr<dcp::Reel> reel) {
977 return static_cast<bool>(reel->main_subtitle());
979 [](shared_ptr<dcp::Reel> reel) {
980 return reel->main_subtitle()->asset()->raw_xml();
982 [](shared_ptr<dcp::Reel> reel) {
983 return reel->main_subtitle()->actual_duration();
988 for (auto i = 0U; i < reels[0]->closed_captions().size(); ++i) {
989 check_text_timing (reels, picture_frame_rate, notes,
990 [i](shared_ptr<dcp::Reel> reel) {
991 return i < reel->closed_captions().size();
993 [i](shared_ptr<dcp::Reel> reel) {
994 return reel->closed_captions()[i]->asset()->raw_xml();
996 [i](shared_ptr<dcp::Reel> reel) {
997 return reel->closed_captions()[i]->actual_duration();
1004 vector<VerificationNote>
1006 vector<boost::filesystem::path> directories,
1007 function<void (string, optional<boost::filesystem::path>)> stage,
1008 function<void (float)> progress,
1009 boost::filesystem::path xsd_dtd_directory
1012 xsd_dtd_directory = boost::filesystem::canonical (xsd_dtd_directory);
1014 vector<VerificationNote> notes;
1017 vector<shared_ptr<DCP>> dcps;
1018 for (auto i: directories) {
1019 dcps.push_back (shared_ptr<DCP> (new DCP (i)));
1022 for (auto dcp: dcps) {
1023 stage ("Checking DCP", dcp->directory());
1026 } catch (ReadError& e) {
1027 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
1028 } catch (XMLError& e) {
1029 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
1030 } catch (MXFFileError& e) {
1031 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
1032 } catch (cxml::Error& e) {
1033 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
1036 if (dcp->standard() != dcp::SMPTE) {
1037 notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::NOT_SMPTE));
1040 for (auto cpl: dcp->cpls()) {
1041 stage ("Checking CPL", cpl->file());
1042 validate_xml (cpl->file().get(), xsd_dtd_directory, notes);
1044 for (auto const& i: cpl->additional_subtitle_languages()) {
1045 verify_language_tag (i, notes);
1048 if (cpl->release_territory()) {
1049 verify_language_tag (cpl->release_territory().get(), notes);
1052 if (dcp->standard() == dcp::SMPTE) {
1053 if (!cpl->annotation_text()) {
1054 notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_ANNOTATION_TEXT_IN_CPL));
1055 } else if (cpl->annotation_text().get() != cpl->content_title_text()) {
1056 notes.push_back (VerificationNote(VerificationNote::VERIFY_WARNING, VerificationNote::CPL_ANNOTATION_TEXT_DIFFERS_FROM_CONTENT_TITLE_TEXT));
1060 /* Check that the CPL's hash corresponds to the PKL */
1061 for (auto i: dcp->pkls()) {
1062 optional<string> h = i->hash(cpl->id());
1063 if (h && make_digest(ArrayData(*cpl->file())) != *h) {
1064 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::CPL_HASH_INCORRECT));
1068 /* set to true if any reel has a MainSubtitle */
1069 auto have_main_subtitle = false;
1070 /* set to true if any reel has no MainSubtitle */
1071 auto have_no_main_subtitle = false;
1072 /* fewest number of closed caption assets seen in a reel */
1073 size_t fewest_closed_captions = SIZE_MAX;
1074 /* most number of closed caption assets seen in a reel */
1075 size_t most_closed_captions = 0;
1077 for (auto reel: cpl->reels()) {
1078 stage ("Checking reel", optional<boost::filesystem::path>());
1080 for (auto i: reel->assets()) {
1081 if (i->duration() && (i->duration().get() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
1082 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::DURATION_TOO_SMALL, i->id()));
1084 if ((i->intrinsic_duration() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
1085 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INTRINSIC_DURATION_TOO_SMALL, i->id()));
1087 auto mxf = dynamic_pointer_cast<ReelMXF>(i);
1088 if (mxf && !mxf->hash()) {
1089 notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_HASH, i->id()});
1093 if (dcp->standard() == dcp::SMPTE) {
1094 boost::optional<int64_t> duration;
1095 for (auto i: reel->assets()) {
1097 duration = i->actual_duration();
1098 } else if (*duration != i->actual_duration()) {
1099 notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISMATCHED_ASSET_DURATION, i->id()));
1105 if (reel->main_picture()) {
1106 /* Check reel stuff */
1107 auto const frame_rate = reel->main_picture()->frame_rate();
1108 if (frame_rate.denominator != 1 ||
1109 (frame_rate.numerator != 24 &&
1110 frame_rate.numerator != 25 &&
1111 frame_rate.numerator != 30 &&
1112 frame_rate.numerator != 48 &&
1113 frame_rate.numerator != 50 &&
1114 frame_rate.numerator != 60 &&
1115 frame_rate.numerator != 96)) {
1116 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INVALID_PICTURE_FRAME_RATE));
1119 if (reel->main_picture()->asset_ref().resolved()) {
1120 verify_main_picture_asset (dcp, reel->main_picture(), stage, progress, notes);
1124 if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
1125 verify_main_sound_asset (dcp, reel->main_sound(), stage, progress, notes);
1128 if (reel->main_subtitle()) {
1129 verify_main_subtitle_reel (reel->main_subtitle(), notes);
1130 if (reel->main_subtitle()->asset_ref().resolved()) {
1131 verify_subtitle_asset (reel->main_subtitle()->asset(), stage, xsd_dtd_directory, notes, state);
1133 have_main_subtitle = true;
1135 have_no_main_subtitle = true;
1138 for (auto i: reel->closed_captions()) {
1139 verify_closed_caption_reel (i, notes);
1140 if (i->asset_ref().resolved()) {
1141 verify_closed_caption_asset (i->asset(), stage, xsd_dtd_directory, notes, state);
1145 fewest_closed_captions = std::min (fewest_closed_captions, reel->closed_captions().size());
1146 most_closed_captions = std::max (most_closed_captions, reel->closed_captions().size());
1149 if (dcp->standard() == dcp::SMPTE) {
1151 if (have_main_subtitle && have_no_main_subtitle) {
1152 notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MAIN_SUBTITLE_NOT_IN_ALL_REELS});
1155 if (fewest_closed_captions != most_closed_captions) {
1156 notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::CLOSED_CAPTION_ASSET_COUNTS_DIFFER});
1159 check_text_timing (cpl->reels(), notes);
1161 LinesCharactersResult result;
1162 for (auto reel: cpl->reels()) {
1163 if (reel->main_subtitle() && reel->main_subtitle()->asset()) {
1164 check_text_lines_and_characters (reel->main_subtitle()->asset(), 52, 79, &result);
1168 if (result.line_count_exceeded) {
1169 notes.push_back (VerificationNote(VerificationNote::VERIFY_WARNING, VerificationNote::TOO_MANY_SUBTITLE_LINES));
1171 if (result.error_length_exceeded) {
1172 notes.push_back (VerificationNote(VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_LINE_TOO_LONG));
1173 } else if (result.warning_length_exceeded) {
1174 notes.push_back (VerificationNote(VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_LINE_LONGER_THAN_RECOMMENDED));
1177 result = LinesCharactersResult();
1178 for (auto reel: cpl->reels()) {
1179 for (auto i: reel->closed_captions()) {
1181 check_text_lines_and_characters (i->asset(), 32, 32, &result);
1186 if (result.line_count_exceeded) {
1187 notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::TOO_MANY_CLOSED_CAPTION_LINES));
1189 if (result.error_length_exceeded) {
1190 notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::CLOSED_CAPTION_LINE_TOO_LONG));
1195 for (auto pkl: dcp->pkls()) {
1196 stage ("Checking PKL", pkl->file());
1197 validate_xml (pkl->file().get(), xsd_dtd_directory, notes);
1200 if (dcp->asset_map_path()) {
1201 stage ("Checking ASSETMAP", dcp->asset_map_path().get());
1202 validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
1204 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::MISSING_ASSETMAP));
1212 dcp::note_to_string (dcp::VerificationNote note)
1214 switch (note.code()) {
1215 case dcp::VerificationNote::GENERAL_READ:
1216 return *note.note();
1217 case dcp::VerificationNote::CPL_HASH_INCORRECT:
1218 return "The hash of the CPL in the PKL does not agree with the CPL file.";
1219 case dcp::VerificationNote::INVALID_PICTURE_FRAME_RATE:
1220 return "The picture in a reel has an invalid frame rate.";
1221 case dcp::VerificationNote::PICTURE_HASH_INCORRECT:
1222 return dcp::String::compose("The hash of the picture asset %1 does not agree with the PKL file.", note.file()->filename());
1223 case dcp::VerificationNote::PKL_CPL_PICTURE_HASHES_DIFFER:
1224 return dcp::String::compose("The PKL and CPL hashes differ for the picture asset %1.", note.file()->filename());
1225 case dcp::VerificationNote::SOUND_HASH_INCORRECT:
1226 return dcp::String::compose("The hash of the sound asset %1 does not agree with the PKL file.", note.file()->filename());
1227 case dcp::VerificationNote::PKL_CPL_SOUND_HASHES_DIFFER:
1228 return dcp::String::compose("The PKL and CPL hashes differ for the sound asset %1.", note.file()->filename());
1229 case dcp::VerificationNote::EMPTY_ASSET_PATH:
1230 return "The asset map contains an empty asset path.";
1231 case dcp::VerificationNote::MISSING_ASSET:
1232 return String::compose("The file for an asset in the asset map cannot be found; missing file is %1.", note.file()->filename());
1233 case dcp::VerificationNote::MISMATCHED_STANDARD:
1234 return "The DCP contains both SMPTE and Interop parts.";
1235 case dcp::VerificationNote::XML_VALIDATION_ERROR:
1236 return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get());
1237 case dcp::VerificationNote::MISSING_ASSETMAP:
1238 return "No ASSETMAP or ASSETMAP.xml was found.";
1239 case dcp::VerificationNote::INTRINSIC_DURATION_TOO_SMALL:
1240 return String::compose("The intrinsic duration of an asset is less than 1 second long: %1", note.note().get());
1241 case dcp::VerificationNote::DURATION_TOO_SMALL:
1242 return String::compose("The duration of an asset is less than 1 second long: %1", note.note().get());
1243 case dcp::VerificationNote::PICTURE_FRAME_TOO_LARGE_IN_BYTES:
1244 return String::compose("The instantaneous bit rate of the picture asset %1 is larger than the limit of 250Mbit/s in at least one place.", note.file()->filename());
1245 case dcp::VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE_IN_BYTES:
1246 return String::compose("The instantaneous bit rate of the picture asset %1 is close to the limit of 250Mbit/s in at least one place.", note.file()->filename());
1247 case dcp::VerificationNote::EXTERNAL_ASSET:
1248 return String::compose("An asset that this DCP refers to is not included in the DCP. It may be a VF. Missing asset is %1.", note.note().get());
1249 case dcp::VerificationNote::NOT_SMPTE:
1250 return "This DCP does not use the SMPTE standard, which is required for Bv2.1 compliance.";
1251 case dcp::VerificationNote::BAD_LANGUAGE:
1252 return String::compose("The DCP specifies a language '%1' which does not conform to the RFC 5646 standard.", note.note().get());
1253 case dcp::VerificationNote::PICTURE_ASSET_INVALID_SIZE_IN_PIXELS:
1254 return String::compose("A picture asset's size (%1) is not one of those allowed by Bv2.1 (2048x858, 1998x1080, 4096x1716 or 3996x2160)", note.note().get());
1255 case dcp::VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_2K:
1256 return String::compose("A picture asset's frame rate (%1) is not one of those allowed for 2K DCPs by Bv2.1 (24, 25 or 48fps)", note.note().get());
1257 case dcp::VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_4K:
1258 return String::compose("A picture asset's frame rate (%1) is not 24fps as required for 4K DCPs by Bv2.1", note.note().get());
1259 case dcp::VerificationNote::PICTURE_ASSET_4K_3D:
1260 return "3D 4K DCPs are not allowed by Bv2.1";
1261 case dcp::VerificationNote::CLOSED_CAPTION_XML_TOO_LARGE_IN_BYTES:
1262 return String::compose("The XML for the closed caption asset %1 is longer than the 256KB maximum required by Bv2.1", note.file()->filename());
1263 case dcp::VerificationNote::TIMED_TEXT_ASSET_TOO_LARGE_IN_BYTES:
1264 return String::compose("The total size of the timed text asset %1 is larger than the 115MB maximum required by Bv2.1", note.file()->filename());
1265 case dcp::VerificationNote::TIMED_TEXT_FONTS_TOO_LARGE_IN_BYTES:
1266 return String::compose("The total size of the fonts in timed text asset %1 is larger than the 10MB maximum required by Bv2.1", note.file()->filename());
1267 case dcp::VerificationNote::MISSING_SUBTITLE_LANGUAGE:
1268 return String::compose("The XML for a SMPTE subtitle asset has no <Language> tag, which is required by Bv2.1", note.file()->filename());
1269 case dcp::VerificationNote::SUBTITLE_LANGUAGES_DIFFER:
1270 return String::compose("Some subtitle assets have different <Language> tags than others", note.file()->filename());
1271 case dcp::VerificationNote::MISSING_SUBTITLE_START_TIME:
1272 return String::compose("The XML for a SMPTE subtitle asset has no <StartTime> tag, which is required by Bv2.1", note.file()->filename());
1273 case dcp::VerificationNote::SUBTITLE_START_TIME_NON_ZERO:
1274 return String::compose("The XML for a SMPTE subtitle asset has a non-zero <StartTime> tag, which is disallowed by Bv2.1", note.file()->filename());
1275 case dcp::VerificationNote::FIRST_TEXT_TOO_EARLY:
1276 return "The first subtitle or closed caption is less than 4 seconds from the start of the DCP.";
1277 case dcp::VerificationNote::SUBTITLE_TOO_SHORT:
1278 return "At least one subtitle is less than the minimum of 15 frames suggested by Bv2.1";
1279 case dcp::VerificationNote::SUBTITLE_TOO_CLOSE:
1280 return "At least one pair of subtitles are separated by less than the the minimum of 2 frames suggested by Bv2.1";
1281 case dcp::VerificationNote::TOO_MANY_SUBTITLE_LINES:
1282 return "There are more than 3 subtitle lines in at least one place in the DCP, which Bv2.1 advises against.";
1283 case dcp::VerificationNote::SUBTITLE_LINE_LONGER_THAN_RECOMMENDED:
1284 return "There are more than 52 characters in at least one subtitle line, which Bv2.1 advises against.";
1285 case dcp::VerificationNote::SUBTITLE_LINE_TOO_LONG:
1286 return "There are more than 79 characters in at least one subtitle line, which Bv2.1 strongly advises against.";
1287 case dcp::VerificationNote::TOO_MANY_CLOSED_CAPTION_LINES:
1288 return "There are more than 3 closed caption lines in at least one place, which is disallowed by Bv2.1";
1289 case dcp::VerificationNote::CLOSED_CAPTION_LINE_TOO_LONG:
1290 return "There are more than 32 characters in at least one closed caption line, which is disallowed by Bv2.1";
1291 case dcp::VerificationNote::INVALID_SOUND_FRAME_RATE:
1292 return "A sound asset has a sampling rate other than 48kHz, which is disallowed by Bv2.1";
1293 case dcp::VerificationNote::MISSING_ANNOTATION_TEXT_IN_CPL:
1294 return "The CPL has no <AnnotationText> tag, which is required by Bv2.1";
1295 case dcp::VerificationNote::CPL_ANNOTATION_TEXT_DIFFERS_FROM_CONTENT_TITLE_TEXT:
1296 return "The CPL's <AnnotationText> differs from its <ContentTitleText>, which Bv2.1 advises against.";
1297 case dcp::VerificationNote::MISMATCHED_ASSET_DURATION:
1298 return "All assets in a reel do not have the same duration, which is required by Bv2.1";
1299 case dcp::VerificationNote::MAIN_SUBTITLE_NOT_IN_ALL_REELS:
1300 return "At least one reel contains a subtitle asset, but some reel(s) do not";
1301 case dcp::VerificationNote::CLOSED_CAPTION_ASSET_COUNTS_DIFFER:
1302 return "At least one reel has closed captions, but reels have different numbers of closed caption assets.";
1303 case dcp::VerificationNote::MISSING_SUBTITLE_ENTRY_POINT:
1304 return "Subtitle assets must have an <EntryPoint> tag.";
1305 case dcp::VerificationNote::SUBTITLE_ENTRY_POINT_NON_ZERO:
1306 return "Subtitle assets must have an <EntryPoint> of 0.";
1307 case dcp::VerificationNote::MISSING_CLOSED_CAPTION_ENTRY_POINT:
1308 return "Closed caption assets must have an <EntryPoint> tag.";
1309 case dcp::VerificationNote::CLOSED_CAPTION_ENTRY_POINT_NON_ZERO:
1310 return "Closed caption assets must have an <EntryPoint> of 0.";
1311 case dcp::VerificationNote::MISSING_HASH:
1312 return String::compose("An asset is missing a <Hash> tag: %1", note.note().get());