2 Copyright (C) 2018-2020 Carl Hetherington <cth@carlh.net>
4 This file is part of libdcp.
6 libdcp is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 libdcp is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with libdcp. If not, see <http://www.gnu.org/licenses/>.
19 In addition, as a special exception, the copyright holders give
20 permission to link the code of portions of this program with the
21 OpenSSL library under certain conditions as described in each
22 individual source file, and distribute linked combinations
25 You must obey the GNU General Public License in all respects
26 for all of the code used other than OpenSSL. If you modify
27 file(s) with this exception, you may extend this exception to your
28 version of the file(s), but you are not obligated to do so. If you
29 do not wish to do so, delete this exception statement from your
30 version. If you delete this exception statement from all source
31 files in the program, then also delete it here.
38 #include "reel_picture_asset.h"
39 #include "reel_sound_asset.h"
40 #include "reel_subtitle_asset.h"
41 #include "interop_subtitle_asset.h"
42 #include "mono_picture_asset.h"
43 #include "mono_picture_frame.h"
44 #include "stereo_picture_asset.h"
45 #include "stereo_picture_frame.h"
46 #include "exceptions.h"
47 #include "compose.hpp"
48 #include "raw_convert.h"
49 #include <xercesc/util/PlatformUtils.hpp>
50 #include <xercesc/parsers/XercesDOMParser.hpp>
51 #include <xercesc/parsers/AbstractDOMParser.hpp>
52 #include <xercesc/sax/HandlerBase.hpp>
53 #include <xercesc/dom/DOMImplementation.hpp>
54 #include <xercesc/dom/DOMImplementationLS.hpp>
55 #include <xercesc/dom/DOMImplementationRegistry.hpp>
56 #include <xercesc/dom/DOMLSParser.hpp>
57 #include <xercesc/dom/DOMException.hpp>
58 #include <xercesc/dom/DOMDocument.hpp>
59 #include <xercesc/dom/DOMNodeList.hpp>
60 #include <xercesc/dom/DOMError.hpp>
61 #include <xercesc/dom/DOMLocator.hpp>
62 #include <xercesc/dom/DOMNamedNodeMap.hpp>
63 #include <xercesc/dom/DOMAttr.hpp>
64 #include <xercesc/dom/DOMErrorHandler.hpp>
65 #include <xercesc/framework/LocalFileInputSource.hpp>
66 #include <xercesc/framework/MemBufInputSource.hpp>
67 #include <boost/noncopyable.hpp>
68 #include <boost/foreach.hpp>
69 #include <boost/algorithm/string.hpp>
81 using boost::shared_ptr;
82 using boost::optional;
83 using boost::function;
84 using boost::dynamic_pointer_cast;
87 using namespace xercesc;
91 xml_ch_to_string (XMLCh const * a)
93 char* x = XMLString::transcode(a);
95 XMLString::release(&x);
99 class XMLValidationError
102 XMLValidationError (SAXParseException const & e)
103 : _message (xml_ch_to_string(e.getMessage()))
104 , _line (e.getLineNumber())
105 , _column (e.getColumnNumber())
110 string message () const {
114 uint64_t line () const {
118 uint64_t column () const {
129 class DCPErrorHandler : public ErrorHandler
132 void warning(const SAXParseException& e)
134 maybe_add (XMLValidationError(e));
137 void error(const SAXParseException& e)
139 maybe_add (XMLValidationError(e));
142 void fatalError(const SAXParseException& e)
144 maybe_add (XMLValidationError(e));
151 list<XMLValidationError> errors () const {
156 void maybe_add (XMLValidationError e)
158 /* XXX: nasty hack */
160 e.message().find("schema document") != string::npos &&
161 e.message().find("has different target namespace from the one specified in instance document") != string::npos
166 _errors.push_back (e);
169 list<XMLValidationError> _errors;
172 class StringToXMLCh : public boost::noncopyable
175 StringToXMLCh (string a)
177 _buffer = XMLString::transcode(a.c_str());
182 XMLString::release (&_buffer);
185 XMLCh const * get () const {
193 class LocalFileResolver : public EntityResolver
196 LocalFileResolver (boost::filesystem::path xsd_dtd_directory)
197 : _xsd_dtd_directory (xsd_dtd_directory)
199 add("http://www.w3.org/2001/XMLSchema.dtd", "XMLSchema.dtd");
200 add("http://www.w3.org/2001/03/xml.xsd", "xml.xsd");
201 add("http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd", "xmldsig-core-schema.xsd");
202 add("http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd", "Main-Stereo-Picture-CPL.xsd");
203 add("http://www.digicine.com/PROTO-ASDCP-CPL-20040511.xsd", "PROTO-ASDCP-CPL-20040511.xsd");
204 add("http://www.digicine.com/PROTO-ASDCP-PKL-20040311.xsd", "PROTO-ASDCP-PKL-20040311.xsd");
205 add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd");
206 add("http://www.digicine.com/PROTO-ASDCP-CC-CPL-20070926#", "PROTO-ASDCP-CC-CPL-20070926.xsd");
207 add("interop-subs", "DCSubtitle.v1.mattsson.xsd");
208 add("http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd", "SMPTE-428-7-2010-DCST.xsd");
209 add("http://www.smpte-ra.org/schemas/429-16/2014/CPL-Metadata", "SMPTE-429-16.xsd");
210 add("http://www.dolby.com/schemas/2012/AD", "Dolby-2012-AD.xsd");
213 InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id)
218 string system_id_str = xml_ch_to_string (system_id);
219 boost::filesystem::path p = _xsd_dtd_directory;
220 if (_files.find(system_id_str) == _files.end()) {
223 p /= _files[system_id_str];
225 StringToXMLCh ch (p.string());
226 return new LocalFileInputSource(ch.get());
230 void add (string uri, string file)
235 std::map<string, string> _files;
236 boost::filesystem::path _xsd_dtd_directory;
241 parse (XercesDOMParser& parser, boost::filesystem::path xml)
243 parser.parse(xml.string().c_str());
248 parse (XercesDOMParser& parser, std::string xml)
250 xercesc::MemBufInputSource buf(reinterpret_cast<unsigned char const*>(xml.c_str()), xml.size(), "");
257 validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, list<VerificationNote>& notes)
260 XMLPlatformUtils::Initialize ();
261 } catch (XMLException& e) {
262 throw MiscError ("Failed to initialise xerces library");
265 DCPErrorHandler error_handler;
267 /* All the xerces objects in this scope must be destroyed before XMLPlatformUtils::Terminate() is called */
269 XercesDOMParser parser;
270 parser.setValidationScheme(XercesDOMParser::Val_Always);
271 parser.setDoNamespaces(true);
272 parser.setDoSchema(true);
274 vector<string> schema;
275 schema.push_back("xmldsig-core-schema.xsd");
276 schema.push_back("SMPTE-429-7-2006-CPL.xsd");
277 schema.push_back("SMPTE-429-8-2006-PKL.xsd");
278 schema.push_back("SMPTE-429-9-2007-AM.xsd");
279 schema.push_back("Main-Stereo-Picture-CPL.xsd");
280 schema.push_back("PROTO-ASDCP-CPL-20040511.xsd");
281 schema.push_back("PROTO-ASDCP-PKL-20040311.xsd");
282 schema.push_back("PROTO-ASDCP-AM-20040311.xsd");
283 schema.push_back("DCSubtitle.v1.mattsson.xsd");
284 schema.push_back("DCDMSubtitle-2010.xsd");
285 schema.push_back("PROTO-ASDCP-CC-CPL-20070926.xsd");
286 schema.push_back("SMPTE-429-16.xsd");
287 schema.push_back("Dolby-2012-AD.xsd");
289 /* XXX: I'm not especially clear what this is for, but it seems to be necessary */
291 BOOST_FOREACH (string i, schema) {
292 locations += String::compose("%1 %1 ", i, i);
295 parser.setExternalSchemaLocation(locations.c_str());
296 parser.setValidationSchemaFullChecking(true);
297 parser.setErrorHandler(&error_handler);
299 LocalFileResolver resolver (xsd_dtd_directory);
300 parser.setEntityResolver(&resolver);
303 parser.resetDocumentPool();
305 } catch (XMLException& e) {
306 throw MiscError(xml_ch_to_string(e.getMessage()));
307 } catch (DOMException& e) {
308 throw MiscError(xml_ch_to_string(e.getMessage()));
310 throw MiscError("Unknown exception from xerces");
314 XMLPlatformUtils::Terminate ();
316 BOOST_FOREACH (XMLValidationError i, error_handler.errors()) {
319 VerificationNote::VERIFY_ERROR,
320 VerificationNote::XML_VALIDATION_ERROR,
330 enum VerifyAssetResult {
331 VERIFY_ASSET_RESULT_GOOD,
332 VERIFY_ASSET_RESULT_CPL_PKL_DIFFER,
333 VERIFY_ASSET_RESULT_BAD
337 static VerifyAssetResult
338 verify_asset (shared_ptr<const DCP> dcp, shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
340 string const actual_hash = reel_mxf->asset_ref()->hash(progress);
342 list<shared_ptr<PKL> > pkls = dcp->pkls();
343 /* We've read this DCP in so it must have at least one PKL */
344 DCP_ASSERT (!pkls.empty());
346 shared_ptr<Asset> asset = reel_mxf->asset_ref().asset();
348 optional<string> pkl_hash;
349 BOOST_FOREACH (shared_ptr<PKL> i, pkls) {
350 pkl_hash = i->hash (reel_mxf->asset_ref()->id());
356 DCP_ASSERT (pkl_hash);
358 optional<string> cpl_hash = reel_mxf->hash();
359 if (cpl_hash && *cpl_hash != *pkl_hash) {
360 return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER;
363 if (actual_hash != *pkl_hash) {
364 return VERIFY_ASSET_RESULT_BAD;
367 return VERIFY_ASSET_RESULT_GOOD;
371 enum VerifyPictureAssetResult
373 VERIFY_PICTURE_ASSET_RESULT_GOOD,
374 VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG,
375 VERIFY_PICTURE_ASSET_RESULT_BAD,
380 biggest_frame_size (shared_ptr<const MonoPictureFrame> frame)
382 return frame->j2k_size ();
386 biggest_frame_size (shared_ptr<const StereoPictureFrame> frame)
388 return max(frame->left_j2k_size(), frame->right_j2k_size());
392 template <class A, class R, class F>
393 optional<VerifyPictureAssetResult>
394 verify_picture_asset_type (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
396 shared_ptr<A> asset = dynamic_pointer_cast<A>(reel_mxf->asset_ref().asset());
398 return optional<VerifyPictureAssetResult>();
401 int biggest_frame = 0;
402 shared_ptr<R> reader = asset->start_read ();
403 int64_t const duration = asset->intrinsic_duration ();
404 for (int64_t i = 0; i < duration; ++i) {
405 shared_ptr<const F> frame = reader->get_frame (i);
406 biggest_frame = max(biggest_frame, biggest_frame_size(frame));
407 progress (float(i) / duration);
410 static const int max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float()));
411 static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float()));
412 if (biggest_frame > max_frame) {
413 return VERIFY_PICTURE_ASSET_RESULT_BAD;
414 } else if (biggest_frame > risky_frame) {
415 return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG;
418 return VERIFY_PICTURE_ASSET_RESULT_GOOD;
422 static VerifyPictureAssetResult
423 verify_picture_asset (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
425 optional<VerifyPictureAssetResult> r = verify_picture_asset_type<MonoPictureAsset, MonoPictureAssetReader, MonoPictureFrame>(reel_mxf, progress);
427 r = verify_picture_asset_type<StereoPictureAsset, StereoPictureAssetReader, StereoPictureFrame>(reel_mxf, progress);
436 verify_main_picture_asset (
437 shared_ptr<const DCP> dcp,
438 shared_ptr<const Reel> reel,
439 function<void (string, optional<boost::filesystem::path>)> stage,
440 function<void (float)> progress,
441 list<VerificationNote>& notes
444 boost::filesystem::path const file = *reel->main_picture()->asset()->file();
445 stage ("Checking picture asset hash", file);
446 VerifyAssetResult const r = verify_asset (dcp, reel->main_picture(), progress);
448 case VERIFY_ASSET_RESULT_BAD:
451 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, file
455 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
458 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE, file
465 stage ("Checking picture frame sizes", reel->main_picture()->asset()->file());
466 VerifyPictureAssetResult const pr = verify_picture_asset (reel->main_picture(), progress);
468 case VERIFY_PICTURE_ASSET_RESULT_BAD:
471 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_FRAME_TOO_LARGE, file
475 case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG:
478 VerificationNote::VERIFY_WARNING, VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE, file
489 verify_main_sound_asset (
490 shared_ptr<const DCP> dcp,
491 shared_ptr<const Reel> reel,
492 function<void (string, optional<boost::filesystem::path>)> stage,
493 function<void (float)> progress,
494 list<VerificationNote>& notes
497 stage ("Checking sound asset hash", reel->main_sound()->asset()->file());
498 VerifyAssetResult const r = verify_asset (dcp, reel->main_sound(), progress);
500 case VERIFY_ASSET_RESULT_BAD:
503 VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *reel->main_sound()->asset()->file()
507 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
510 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE, *reel->main_sound()->asset()->file()
521 verify_main_subtitle_asset (
522 shared_ptr<const Reel> reel,
523 function<void (string, optional<boost::filesystem::path>)> stage,
524 boost::filesystem::path xsd_dtd_directory,
525 list<VerificationNote>& notes
528 shared_ptr<ReelSubtitleAsset> reel_asset = reel->main_subtitle ();
529 stage ("Checking subtitle XML", reel->main_subtitle()->asset()->file());
530 /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk
531 * gets passed through libdcp which may clean up and therefore hide errors.
533 validate_xml (reel->main_subtitle()->asset()->raw_xml(), xsd_dtd_directory, notes);
537 list<VerificationNote>
539 vector<boost::filesystem::path> directories,
540 function<void (string, optional<boost::filesystem::path>)> stage,
541 function<void (float)> progress,
542 boost::filesystem::path xsd_dtd_directory
545 xsd_dtd_directory = boost::filesystem::canonical (xsd_dtd_directory);
547 list<VerificationNote> notes;
549 list<shared_ptr<DCP> > dcps;
550 BOOST_FOREACH (boost::filesystem::path i, directories) {
551 dcps.push_back (shared_ptr<DCP> (new DCP (i)));
554 BOOST_FOREACH (shared_ptr<DCP> dcp, dcps) {
555 stage ("Checking DCP", dcp->directory());
558 } catch (ReadError& e) {
559 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
560 } catch (XMLError& e) {
561 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
562 } catch (cxml::Error& e) {
563 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
566 BOOST_FOREACH (shared_ptr<CPL> cpl, dcp->cpls()) {
567 stage ("Checking CPL", cpl->file());
568 validate_xml (cpl->file().get(), xsd_dtd_directory, notes);
570 /* Check that the CPL's hash corresponds to the PKL */
571 BOOST_FOREACH (shared_ptr<PKL> i, dcp->pkls()) {
572 optional<string> h = i->hash(cpl->id());
573 if (h && make_digest(Data(*cpl->file())) != *h) {
574 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::CPL_HASH_INCORRECT));
578 BOOST_FOREACH (shared_ptr<Reel> reel, cpl->reels()) {
579 stage ("Checking reel", optional<boost::filesystem::path>());
581 BOOST_FOREACH (shared_ptr<ReelAsset> i, reel->assets()) {
582 if (i->duration() && (i->duration().get() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
583 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::DURATION_TOO_SMALL, i->id()));
585 if ((i->intrinsic_duration() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
586 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INTRINSIC_DURATION_TOO_SMALL, i->id()));
590 if (reel->main_picture()) {
591 /* Check reel stuff */
592 Fraction const frame_rate = reel->main_picture()->frame_rate();
593 if (frame_rate.denominator != 1 ||
594 (frame_rate.numerator != 24 &&
595 frame_rate.numerator != 25 &&
596 frame_rate.numerator != 30 &&
597 frame_rate.numerator != 48 &&
598 frame_rate.numerator != 50 &&
599 frame_rate.numerator != 60 &&
600 frame_rate.numerator != 96)) {
601 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INVALID_PICTURE_FRAME_RATE));
604 if (reel->main_picture()->asset_ref().resolved()) {
605 verify_main_picture_asset (dcp, reel, stage, progress, notes);
609 if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
610 verify_main_sound_asset (dcp, reel, stage, progress, notes);
613 if (reel->main_subtitle() && reel->main_subtitle()->asset_ref().resolved()) {
614 verify_main_subtitle_asset (reel, stage, xsd_dtd_directory, notes);
619 BOOST_FOREACH (shared_ptr<PKL> pkl, dcp->pkls()) {
620 stage ("Checking PKL", pkl->file());
621 validate_xml (pkl->file().get(), xsd_dtd_directory, notes);
624 if (dcp->asset_map_path()) {
625 stage ("Checking ASSETMAP", dcp->asset_map_path().get());
626 validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
628 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::MISSING_ASSETMAP));
636 dcp::note_to_string (dcp::VerificationNote note)
638 switch (note.code()) {
639 case dcp::VerificationNote::GENERAL_READ:
641 case dcp::VerificationNote::CPL_HASH_INCORRECT:
642 return "The hash of the CPL in the PKL does not agree with the CPL file.";
643 case dcp::VerificationNote::INVALID_PICTURE_FRAME_RATE:
644 return "The picture in a reel has an invalid frame rate.";
645 case dcp::VerificationNote::PICTURE_HASH_INCORRECT:
646 return dcp::String::compose("The hash of the picture asset %1 does not agree with the PKL file.", note.file()->filename());
647 case dcp::VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE:
648 return dcp::String::compose("The PKL and CPL hashes disagree for the picture asset %1.", note.file()->filename());
649 case dcp::VerificationNote::SOUND_HASH_INCORRECT:
650 return dcp::String::compose("The hash of the sound asset %1 does not agree with the PKL file.", note.file()->filename());
651 case dcp::VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE:
652 return dcp::String::compose("The PKL and CPL hashes disagree for the sound asset %1.", note.file()->filename());
653 case dcp::VerificationNote::EMPTY_ASSET_PATH:
654 return "The asset map contains an empty asset path.";
655 case dcp::VerificationNote::MISSING_ASSET:
656 return String::compose("The file for an asset in the asset map cannot be found; missing file is %1.", note.file()->filename());
657 case dcp::VerificationNote::MISMATCHED_STANDARD:
658 return "The DCP contains both SMPTE and Interop parts.";
659 case dcp::VerificationNote::XML_VALIDATION_ERROR:
660 return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get());
661 case dcp::VerificationNote::MISSING_ASSETMAP:
662 return "No ASSETMAP or ASSETMAP.xml was found.";
663 case dcp::VerificationNote::INTRINSIC_DURATION_TOO_SMALL:
664 return String::compose("The intrinsic duration of an asset is less than 1 second long: %1", note.note().get());
665 case dcp::VerificationNote::DURATION_TOO_SMALL:
666 return String::compose("The duration of an asset is less than 1 second long: %1", note.note().get());
667 case dcp::VerificationNote::PICTURE_FRAME_TOO_LARGE:
668 return String::compose("The instantaneous bit rate of the picture asset %1 is larger than the limit of 250Mbit/s in at least one place.", note.file()->filename());
669 case dcp::VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE:
670 return String::compose("The instantaneous bit rate of the picture asset %1 is close to the limit of 250Mbit/s in at least one place.", note.file()->filename());
671 case dcp::VerificationNote::EXTERNAL_ASSET:
672 return String::compose("An asset that this DCP refers to is not included in the DCP. It may be a VF. Missing asset is %1.", note.note().get());