2 Copyright (C) 2018-2020 Carl Hetherington <cth@carlh.net>
4 This file is part of libdcp.
6 libdcp is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 libdcp is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with libdcp. If not, see <http://www.gnu.org/licenses/>.
19 In addition, as a special exception, the copyright holders give
20 permission to link the code of portions of this program with the
21 OpenSSL library under certain conditions as described in each
22 individual source file, and distribute linked combinations
25 You must obey the GNU General Public License in all respects
26 for all of the code used other than OpenSSL. If you modify
27 file(s) with this exception, you may extend this exception to your
28 version of the file(s), but you are not obligated to do so. If you
29 do not wish to do so, delete this exception statement from your
30 version. If you delete this exception statement from all source
31 files in the program, then also delete it here.
38 #include "reel_picture_asset.h"
39 #include "reel_sound_asset.h"
40 #include "reel_subtitle_asset.h"
41 #include "interop_subtitle_asset.h"
42 #include "mono_picture_asset.h"
43 #include "mono_picture_frame.h"
44 #include "stereo_picture_asset.h"
45 #include "stereo_picture_frame.h"
46 #include "exceptions.h"
47 #include "compose.hpp"
48 #include "raw_convert.h"
49 #include <xercesc/util/PlatformUtils.hpp>
50 #include <xercesc/parsers/XercesDOMParser.hpp>
51 #include <xercesc/parsers/AbstractDOMParser.hpp>
52 #include <xercesc/sax/HandlerBase.hpp>
53 #include <xercesc/dom/DOMImplementation.hpp>
54 #include <xercesc/dom/DOMImplementationLS.hpp>
55 #include <xercesc/dom/DOMImplementationRegistry.hpp>
56 #include <xercesc/dom/DOMLSParser.hpp>
57 #include <xercesc/dom/DOMException.hpp>
58 #include <xercesc/dom/DOMDocument.hpp>
59 #include <xercesc/dom/DOMNodeList.hpp>
60 #include <xercesc/dom/DOMError.hpp>
61 #include <xercesc/dom/DOMLocator.hpp>
62 #include <xercesc/dom/DOMNamedNodeMap.hpp>
63 #include <xercesc/dom/DOMAttr.hpp>
64 #include <xercesc/dom/DOMErrorHandler.hpp>
65 #include <xercesc/framework/LocalFileInputSource.hpp>
66 #include <xercesc/framework/MemBufInputSource.hpp>
67 #include <boost/noncopyable.hpp>
68 #include <boost/foreach.hpp>
69 #include <boost/algorithm/string.hpp>
81 using boost::shared_ptr;
82 using boost::optional;
83 using boost::function;
84 using boost::dynamic_pointer_cast;
87 using namespace xercesc;
91 xml_ch_to_string (XMLCh const * a)
93 char* x = XMLString::transcode(a);
95 XMLString::release(&x);
99 class XMLValidationError
102 XMLValidationError (SAXParseException const & e)
103 : _message (xml_ch_to_string(e.getMessage()))
104 , _line (e.getLineNumber())
105 , _column (e.getColumnNumber())
110 string message () const {
114 uint64_t line () const {
118 uint64_t column () const {
129 class DCPErrorHandler : public ErrorHandler
132 void warning(const SAXParseException& e)
134 maybe_add (XMLValidationError(e));
137 void error(const SAXParseException& e)
139 maybe_add (XMLValidationError(e));
142 void fatalError(const SAXParseException& e)
144 maybe_add (XMLValidationError(e));
151 list<XMLValidationError> errors () const {
156 void maybe_add (XMLValidationError e)
158 /* XXX: nasty hack */
160 e.message().find("schema document") != string::npos &&
161 e.message().find("has different target namespace from the one specified in instance document") != string::npos
166 _errors.push_back (e);
169 list<XMLValidationError> _errors;
172 class StringToXMLCh : public boost::noncopyable
175 StringToXMLCh (string a)
177 _buffer = XMLString::transcode(a.c_str());
182 XMLString::release (&_buffer);
185 XMLCh const * get () const {
193 class LocalFileResolver : public EntityResolver
196 LocalFileResolver (boost::filesystem::path xsd_dtd_directory)
197 : _xsd_dtd_directory (xsd_dtd_directory)
199 add("http://www.w3.org/2001/XMLSchema.dtd", "XMLSchema.dtd");
200 add("http://www.w3.org/2001/03/xml.xsd", "xml.xsd");
201 add("http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd", "xmldsig-core-schema.xsd");
202 add("http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd", "Main-Stereo-Picture-CPL.xsd");
203 add("http://www.digicine.com/PROTO-ASDCP-CPL-20040511.xsd", "PROTO-ASDCP-CPL-20040511.xsd");
204 add("http://www.digicine.com/PROTO-ASDCP-PKL-20040311.xsd", "PROTO-ASDCP-PKL-20040311.xsd");
205 add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd");
206 add("http://www.digicine.com/PROTO-ASDCP-CC-CPL-20070926#", "PROTO-ASDCP-CC-CPL-20070926.xsd");
207 add("interop-subs", "DCSubtitle.v1.mattsson.xsd");
208 add("http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd", "SMPTE-428-7-2010-DCST.xsd");
209 add("http://www.smpte-ra.org/schemas/429-16/2014/CPL-Metadata", "SMPTE-429-16.xsd");
210 add("http://www.dolby.com/schemas/2012/AD", "Dolby-2012-AD.xsd");
211 add("http://www.smpte-ra.org/schemas/429-10/2008/Main-Stereo-Picture-CPL", "SMPTE-429-10-2008.xsd");
214 InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id)
219 string system_id_str = xml_ch_to_string (system_id);
220 boost::filesystem::path p = _xsd_dtd_directory;
221 if (_files.find(system_id_str) == _files.end()) {
224 p /= _files[system_id_str];
226 StringToXMLCh ch (p.string());
227 return new LocalFileInputSource(ch.get());
231 void add (string uri, string file)
236 std::map<string, string> _files;
237 boost::filesystem::path _xsd_dtd_directory;
242 parse (XercesDOMParser& parser, boost::filesystem::path xml)
244 parser.parse(xml.string().c_str());
249 parse (XercesDOMParser& parser, std::string xml)
251 xercesc::MemBufInputSource buf(reinterpret_cast<unsigned char const*>(xml.c_str()), xml.size(), "");
258 validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, list<VerificationNote>& notes)
261 XMLPlatformUtils::Initialize ();
262 } catch (XMLException& e) {
263 throw MiscError ("Failed to initialise xerces library");
266 DCPErrorHandler error_handler;
268 /* All the xerces objects in this scope must be destroyed before XMLPlatformUtils::Terminate() is called */
270 XercesDOMParser parser;
271 parser.setValidationScheme(XercesDOMParser::Val_Always);
272 parser.setDoNamespaces(true);
273 parser.setDoSchema(true);
275 vector<string> schema;
276 schema.push_back("xmldsig-core-schema.xsd");
277 schema.push_back("SMPTE-429-7-2006-CPL.xsd");
278 schema.push_back("SMPTE-429-8-2006-PKL.xsd");
279 schema.push_back("SMPTE-429-9-2007-AM.xsd");
280 schema.push_back("Main-Stereo-Picture-CPL.xsd");
281 schema.push_back("PROTO-ASDCP-CPL-20040511.xsd");
282 schema.push_back("PROTO-ASDCP-PKL-20040311.xsd");
283 schema.push_back("PROTO-ASDCP-AM-20040311.xsd");
284 schema.push_back("DCSubtitle.v1.mattsson.xsd");
285 schema.push_back("DCDMSubtitle-2010.xsd");
286 schema.push_back("PROTO-ASDCP-CC-CPL-20070926.xsd");
287 schema.push_back("SMPTE-429-16.xsd");
288 schema.push_back("Dolby-2012-AD.xsd");
289 schema.push_back("SMPTE-429-10-2008.xsd");
291 /* XXX: I'm not especially clear what this is for, but it seems to be necessary */
293 BOOST_FOREACH (string i, schema) {
294 locations += String::compose("%1 %1 ", i, i);
297 parser.setExternalSchemaLocation(locations.c_str());
298 parser.setValidationSchemaFullChecking(true);
299 parser.setErrorHandler(&error_handler);
301 LocalFileResolver resolver (xsd_dtd_directory);
302 parser.setEntityResolver(&resolver);
305 parser.resetDocumentPool();
307 } catch (XMLException& e) {
308 throw MiscError(xml_ch_to_string(e.getMessage()));
309 } catch (DOMException& e) {
310 throw MiscError(xml_ch_to_string(e.getMessage()));
312 throw MiscError("Unknown exception from xerces");
316 XMLPlatformUtils::Terminate ();
318 BOOST_FOREACH (XMLValidationError i, error_handler.errors()) {
321 VerificationNote::VERIFY_ERROR,
322 VerificationNote::XML_VALIDATION_ERROR,
332 enum VerifyAssetResult {
333 VERIFY_ASSET_RESULT_GOOD,
334 VERIFY_ASSET_RESULT_CPL_PKL_DIFFER,
335 VERIFY_ASSET_RESULT_BAD
339 static VerifyAssetResult
340 verify_asset (shared_ptr<const DCP> dcp, shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
342 string const actual_hash = reel_mxf->asset_ref()->hash(progress);
344 list<shared_ptr<PKL> > pkls = dcp->pkls();
345 /* We've read this DCP in so it must have at least one PKL */
346 DCP_ASSERT (!pkls.empty());
348 shared_ptr<Asset> asset = reel_mxf->asset_ref().asset();
350 optional<string> pkl_hash;
351 BOOST_FOREACH (shared_ptr<PKL> i, pkls) {
352 pkl_hash = i->hash (reel_mxf->asset_ref()->id());
358 DCP_ASSERT (pkl_hash);
360 optional<string> cpl_hash = reel_mxf->hash();
361 if (cpl_hash && *cpl_hash != *pkl_hash) {
362 return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER;
365 if (actual_hash != *pkl_hash) {
366 return VERIFY_ASSET_RESULT_BAD;
369 return VERIFY_ASSET_RESULT_GOOD;
373 enum VerifyPictureAssetResult
375 VERIFY_PICTURE_ASSET_RESULT_GOOD,
376 VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG,
377 VERIFY_PICTURE_ASSET_RESULT_BAD,
382 biggest_frame_size (shared_ptr<const MonoPictureFrame> frame)
384 return frame->j2k_size ();
388 biggest_frame_size (shared_ptr<const StereoPictureFrame> frame)
390 return max(frame->left_j2k_size(), frame->right_j2k_size());
394 template <class A, class R, class F>
395 optional<VerifyPictureAssetResult>
396 verify_picture_asset_type (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
398 shared_ptr<A> asset = dynamic_pointer_cast<A>(reel_mxf->asset_ref().asset());
400 return optional<VerifyPictureAssetResult>();
403 int biggest_frame = 0;
404 shared_ptr<R> reader = asset->start_read ();
405 int64_t const duration = asset->intrinsic_duration ();
406 for (int64_t i = 0; i < duration; ++i) {
407 shared_ptr<const F> frame = reader->get_frame (i);
408 biggest_frame = max(biggest_frame, biggest_frame_size(frame));
409 progress (float(i) / duration);
412 static const int max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float()));
413 static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float()));
414 if (biggest_frame > max_frame) {
415 return VERIFY_PICTURE_ASSET_RESULT_BAD;
416 } else if (biggest_frame > risky_frame) {
417 return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG;
420 return VERIFY_PICTURE_ASSET_RESULT_GOOD;
424 static VerifyPictureAssetResult
425 verify_picture_asset (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
427 optional<VerifyPictureAssetResult> r = verify_picture_asset_type<MonoPictureAsset, MonoPictureAssetReader, MonoPictureFrame>(reel_mxf, progress);
429 r = verify_picture_asset_type<StereoPictureAsset, StereoPictureAssetReader, StereoPictureFrame>(reel_mxf, progress);
438 verify_main_picture_asset (
439 shared_ptr<const DCP> dcp,
440 shared_ptr<const Reel> reel,
441 function<void (string, optional<boost::filesystem::path>)> stage,
442 function<void (float)> progress,
443 list<VerificationNote>& notes
446 boost::filesystem::path const file = *reel->main_picture()->asset()->file();
447 stage ("Checking picture asset hash", file);
448 VerifyAssetResult const r = verify_asset (dcp, reel->main_picture(), progress);
450 case VERIFY_ASSET_RESULT_BAD:
453 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, file
457 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
460 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE, file
467 stage ("Checking picture frame sizes", reel->main_picture()->asset()->file());
468 VerifyPictureAssetResult const pr = verify_picture_asset (reel->main_picture(), progress);
470 case VERIFY_PICTURE_ASSET_RESULT_BAD:
473 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_FRAME_TOO_LARGE, file
477 case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG:
480 VerificationNote::VERIFY_WARNING, VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE, file
491 verify_main_sound_asset (
492 shared_ptr<const DCP> dcp,
493 shared_ptr<const Reel> reel,
494 function<void (string, optional<boost::filesystem::path>)> stage,
495 function<void (float)> progress,
496 list<VerificationNote>& notes
499 stage ("Checking sound asset hash", reel->main_sound()->asset()->file());
500 VerifyAssetResult const r = verify_asset (dcp, reel->main_sound(), progress);
502 case VERIFY_ASSET_RESULT_BAD:
505 VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *reel->main_sound()->asset()->file()
509 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
512 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE, *reel->main_sound()->asset()->file()
523 verify_main_subtitle_asset (
524 shared_ptr<const Reel> reel,
525 function<void (string, optional<boost::filesystem::path>)> stage,
526 boost::filesystem::path xsd_dtd_directory,
527 list<VerificationNote>& notes
530 shared_ptr<ReelSubtitleAsset> reel_asset = reel->main_subtitle ();
531 stage ("Checking subtitle XML", reel->main_subtitle()->asset()->file());
532 /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk
533 * gets passed through libdcp which may clean up and therefore hide errors.
535 validate_xml (reel->main_subtitle()->asset()->raw_xml(), xsd_dtd_directory, notes);
539 list<VerificationNote>
541 vector<boost::filesystem::path> directories,
542 function<void (string, optional<boost::filesystem::path>)> stage,
543 function<void (float)> progress,
544 boost::filesystem::path xsd_dtd_directory
547 xsd_dtd_directory = boost::filesystem::canonical (xsd_dtd_directory);
549 list<VerificationNote> notes;
551 list<shared_ptr<DCP> > dcps;
552 BOOST_FOREACH (boost::filesystem::path i, directories) {
553 dcps.push_back (shared_ptr<DCP> (new DCP (i)));
556 BOOST_FOREACH (shared_ptr<DCP> dcp, dcps) {
557 stage ("Checking DCP", dcp->directory());
560 } catch (ReadError& e) {
561 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
562 } catch (XMLError& e) {
563 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
564 } catch (cxml::Error& e) {
565 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
568 BOOST_FOREACH (shared_ptr<CPL> cpl, dcp->cpls()) {
569 stage ("Checking CPL", cpl->file());
570 validate_xml (cpl->file().get(), xsd_dtd_directory, notes);
572 /* Check that the CPL's hash corresponds to the PKL */
573 BOOST_FOREACH (shared_ptr<PKL> i, dcp->pkls()) {
574 optional<string> h = i->hash(cpl->id());
575 if (h && make_digest(Data(*cpl->file())) != *h) {
576 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::CPL_HASH_INCORRECT));
580 BOOST_FOREACH (shared_ptr<Reel> reel, cpl->reels()) {
581 stage ("Checking reel", optional<boost::filesystem::path>());
583 BOOST_FOREACH (shared_ptr<ReelAsset> i, reel->assets()) {
584 if (i->duration() && (i->duration().get() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
585 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::DURATION_TOO_SMALL, i->id()));
587 if ((i->intrinsic_duration() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
588 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INTRINSIC_DURATION_TOO_SMALL, i->id()));
592 if (reel->main_picture()) {
593 /* Check reel stuff */
594 Fraction const frame_rate = reel->main_picture()->frame_rate();
595 if (frame_rate.denominator != 1 ||
596 (frame_rate.numerator != 24 &&
597 frame_rate.numerator != 25 &&
598 frame_rate.numerator != 30 &&
599 frame_rate.numerator != 48 &&
600 frame_rate.numerator != 50 &&
601 frame_rate.numerator != 60 &&
602 frame_rate.numerator != 96)) {
603 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INVALID_PICTURE_FRAME_RATE));
606 if (reel->main_picture()->asset_ref().resolved()) {
607 verify_main_picture_asset (dcp, reel, stage, progress, notes);
611 if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
612 verify_main_sound_asset (dcp, reel, stage, progress, notes);
615 if (reel->main_subtitle() && reel->main_subtitle()->asset_ref().resolved()) {
616 verify_main_subtitle_asset (reel, stage, xsd_dtd_directory, notes);
621 BOOST_FOREACH (shared_ptr<PKL> pkl, dcp->pkls()) {
622 stage ("Checking PKL", pkl->file());
623 validate_xml (pkl->file().get(), xsd_dtd_directory, notes);
626 if (dcp->asset_map_path()) {
627 stage ("Checking ASSETMAP", dcp->asset_map_path().get());
628 validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
630 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::MISSING_ASSETMAP));
638 dcp::note_to_string (dcp::VerificationNote note)
640 switch (note.code()) {
641 case dcp::VerificationNote::GENERAL_READ:
643 case dcp::VerificationNote::CPL_HASH_INCORRECT:
644 return "The hash of the CPL in the PKL does not agree with the CPL file.";
645 case dcp::VerificationNote::INVALID_PICTURE_FRAME_RATE:
646 return "The picture in a reel has an invalid frame rate.";
647 case dcp::VerificationNote::PICTURE_HASH_INCORRECT:
648 return dcp::String::compose("The hash of the picture asset %1 does not agree with the PKL file.", note.file()->filename());
649 case dcp::VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE:
650 return dcp::String::compose("The PKL and CPL hashes disagree for the picture asset %1.", note.file()->filename());
651 case dcp::VerificationNote::SOUND_HASH_INCORRECT:
652 return dcp::String::compose("The hash of the sound asset %1 does not agree with the PKL file.", note.file()->filename());
653 case dcp::VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE:
654 return dcp::String::compose("The PKL and CPL hashes disagree for the sound asset %1.", note.file()->filename());
655 case dcp::VerificationNote::EMPTY_ASSET_PATH:
656 return "The asset map contains an empty asset path.";
657 case dcp::VerificationNote::MISSING_ASSET:
658 return String::compose("The file for an asset in the asset map cannot be found; missing file is %1.", note.file()->filename());
659 case dcp::VerificationNote::MISMATCHED_STANDARD:
660 return "The DCP contains both SMPTE and Interop parts.";
661 case dcp::VerificationNote::XML_VALIDATION_ERROR:
662 return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get());
663 case dcp::VerificationNote::MISSING_ASSETMAP:
664 return "No ASSETMAP or ASSETMAP.xml was found.";
665 case dcp::VerificationNote::INTRINSIC_DURATION_TOO_SMALL:
666 return String::compose("The intrinsic duration of an asset is less than 1 second long: %1", note.note().get());
667 case dcp::VerificationNote::DURATION_TOO_SMALL:
668 return String::compose("The duration of an asset is less than 1 second long: %1", note.note().get());
669 case dcp::VerificationNote::PICTURE_FRAME_TOO_LARGE:
670 return String::compose("The instantaneous bit rate of the picture asset %1 is larger than the limit of 250Mbit/s in at least one place.", note.file()->filename());
671 case dcp::VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE:
672 return String::compose("The instantaneous bit rate of the picture asset %1 is close to the limit of 250Mbit/s in at least one place.", note.file()->filename());
673 case dcp::VerificationNote::EXTERNAL_ASSET:
674 return String::compose("An asset that this DCP refers to is not included in the DCP. It may be a VF. Missing asset is %1.", note.note().get());