2 Copyright (C) 2018-2020 Carl Hetherington <cth@carlh.net>
4 This file is part of libdcp.
6 libdcp is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 libdcp is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with libdcp. If not, see <http://www.gnu.org/licenses/>.
19 In addition, as a special exception, the copyright holders give
20 permission to link the code of portions of this program with the
21 OpenSSL library under certain conditions as described in each
22 individual source file, and distribute linked combinations
25 You must obey the GNU General Public License in all respects
26 for all of the code used other than OpenSSL. If you modify
27 file(s) with this exception, you may extend this exception to your
28 version of the file(s), but you are not obligated to do so. If you
29 do not wish to do so, delete this exception statement from your
30 version. If you delete this exception statement from all source
31 files in the program, then also delete it here.
38 #include "reel_picture_asset.h"
39 #include "reel_sound_asset.h"
40 #include "reel_subtitle_asset.h"
41 #include "interop_subtitle_asset.h"
42 #include "mono_picture_asset.h"
43 #include "mono_picture_frame.h"
44 #include "stereo_picture_asset.h"
45 #include "stereo_picture_frame.h"
46 #include "exceptions.h"
47 #include "compose.hpp"
48 #include "raw_convert.h"
49 #include <xercesc/util/PlatformUtils.hpp>
50 #include <xercesc/parsers/XercesDOMParser.hpp>
51 #include <xercesc/parsers/AbstractDOMParser.hpp>
52 #include <xercesc/sax/HandlerBase.hpp>
53 #include <xercesc/dom/DOMImplementation.hpp>
54 #include <xercesc/dom/DOMImplementationLS.hpp>
55 #include <xercesc/dom/DOMImplementationRegistry.hpp>
56 #include <xercesc/dom/DOMLSParser.hpp>
57 #include <xercesc/dom/DOMException.hpp>
58 #include <xercesc/dom/DOMDocument.hpp>
59 #include <xercesc/dom/DOMNodeList.hpp>
60 #include <xercesc/dom/DOMError.hpp>
61 #include <xercesc/dom/DOMLocator.hpp>
62 #include <xercesc/dom/DOMNamedNodeMap.hpp>
63 #include <xercesc/dom/DOMAttr.hpp>
64 #include <xercesc/dom/DOMErrorHandler.hpp>
65 #include <xercesc/framework/LocalFileInputSource.hpp>
66 #include <xercesc/framework/MemBufInputSource.hpp>
67 #include <boost/noncopyable.hpp>
68 #include <boost/foreach.hpp>
69 #include <boost/algorithm/string.hpp>
81 using boost::shared_ptr;
82 using boost::optional;
83 using boost::function;
84 using boost::dynamic_pointer_cast;
87 using namespace xercesc;
91 xml_ch_to_string (XMLCh const * a)
93 char* x = XMLString::transcode(a);
95 XMLString::release(&x);
99 class XMLValidationError
102 XMLValidationError (SAXParseException const & e)
103 : _message (xml_ch_to_string(e.getMessage()))
104 , _line (e.getLineNumber())
105 , _column (e.getColumnNumber())
110 string message () const {
114 uint64_t line () const {
118 uint64_t column () const {
129 class DCPErrorHandler : public ErrorHandler
132 void warning(const SAXParseException& e)
134 maybe_add (XMLValidationError(e));
137 void error(const SAXParseException& e)
139 maybe_add (XMLValidationError(e));
142 void fatalError(const SAXParseException& e)
144 maybe_add (XMLValidationError(e));
151 list<XMLValidationError> errors () const {
156 void maybe_add (XMLValidationError e)
158 /* XXX: nasty hack */
160 e.message().find("schema document") != string::npos &&
161 e.message().find("has different target namespace from the one specified in instance document") != string::npos
166 _errors.push_back (e);
169 list<XMLValidationError> _errors;
172 class StringToXMLCh : public boost::noncopyable
175 StringToXMLCh (string a)
177 _buffer = XMLString::transcode(a.c_str());
182 XMLString::release (&_buffer);
185 XMLCh const * get () const {
193 class LocalFileResolver : public EntityResolver
196 LocalFileResolver (boost::filesystem::path xsd_dtd_directory)
197 : _xsd_dtd_directory (xsd_dtd_directory)
199 add("http://www.w3.org/2001/XMLSchema.dtd", "XMLSchema.dtd");
200 add("http://www.w3.org/2001/03/xml.xsd", "xml.xsd");
201 add("http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd", "xmldsig-core-schema.xsd");
202 add("http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd", "Main-Stereo-Picture-CPL.xsd");
203 add("http://www.digicine.com/PROTO-ASDCP-CPL-20040511.xsd", "PROTO-ASDCP-CPL-20040511.xsd");
204 add("http://www.digicine.com/PROTO-ASDCP-PKL-20040311.xsd", "PROTO-ASDCP-PKL-20040311.xsd");
205 add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd");
206 add("interop-subs", "DCSubtitle.v1.mattsson.xsd");
207 add("http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd", "SMPTE-428-7-2010-DCST.xsd");
210 InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id)
212 string system_id_str = xml_ch_to_string (system_id);
213 boost::filesystem::path p = _xsd_dtd_directory;
214 if (_files.find(system_id_str) == _files.end()) {
217 p /= _files[system_id_str];
219 StringToXMLCh ch (p.string());
220 return new LocalFileInputSource(ch.get());
224 void add (string uri, string file)
229 std::map<string, string> _files;
230 boost::filesystem::path _xsd_dtd_directory;
235 parse (XercesDOMParser& parser, boost::filesystem::path xml)
237 parser.parse(xml.string().c_str());
242 parse (XercesDOMParser& parser, std::string xml)
244 xercesc::MemBufInputSource buf(reinterpret_cast<unsigned char const*>(xml.c_str()), xml.size(), "");
251 validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, list<VerificationNote>& notes)
254 XMLPlatformUtils::Initialize ();
255 } catch (XMLException& e) {
256 throw MiscError ("Failed to initialise xerces library");
259 DCPErrorHandler error_handler;
261 /* All the xerces objects in this scope must be destroyed before XMLPlatformUtils::Terminate() is called */
263 XercesDOMParser parser;
264 parser.setValidationScheme(XercesDOMParser::Val_Always);
265 parser.setDoNamespaces(true);
266 parser.setDoSchema(true);
268 vector<string> schema;
269 schema.push_back("xmldsig-core-schema.xsd");
270 schema.push_back("SMPTE-429-7-2006-CPL.xsd");
271 schema.push_back("SMPTE-429-8-2006-PKL.xsd");
272 schema.push_back("SMPTE-429-9-2007-AM.xsd");
273 schema.push_back("Main-Stereo-Picture-CPL.xsd");
274 schema.push_back("PROTO-ASDCP-CPL-20040511.xsd");
275 schema.push_back("PROTO-ASDCP-PKL-20040311.xsd");
276 schema.push_back("PROTO-ASDCP-AM-20040311.xsd");
277 schema.push_back("DCSubtitle.v1.mattsson.xsd");
278 schema.push_back("DCDMSubtitle-2010.xsd");
280 /* XXX: I'm not especially clear what this is for, but it seems to be necessary */
282 BOOST_FOREACH (string i, schema) {
283 locations += String::compose("%1 %1 ", i, i);
286 parser.setExternalSchemaLocation(locations.c_str());
287 parser.setValidationSchemaFullChecking(true);
288 parser.setErrorHandler(&error_handler);
290 LocalFileResolver resolver (xsd_dtd_directory);
291 parser.setEntityResolver(&resolver);
294 parser.resetDocumentPool();
296 } catch (XMLException& e) {
297 throw MiscError(xml_ch_to_string(e.getMessage()));
298 } catch (DOMException& e) {
299 throw MiscError(xml_ch_to_string(e.getMessage()));
301 throw MiscError("Unknown exception from xerces");
305 XMLPlatformUtils::Terminate ();
307 BOOST_FOREACH (XMLValidationError i, error_handler.errors()) {
310 VerificationNote::VERIFY_ERROR,
311 VerificationNote::XML_VALIDATION_ERROR,
321 enum VerifyAssetResult {
322 VERIFY_ASSET_RESULT_GOOD,
323 VERIFY_ASSET_RESULT_CPL_PKL_DIFFER,
324 VERIFY_ASSET_RESULT_BAD
328 static VerifyAssetResult
329 verify_asset (shared_ptr<const DCP> dcp, shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
331 string const actual_hash = reel_mxf->asset_ref()->hash(progress);
333 list<shared_ptr<PKL> > pkls = dcp->pkls();
334 /* We've read this DCP in so it must have at least one PKL */
335 DCP_ASSERT (!pkls.empty());
337 shared_ptr<Asset> asset = reel_mxf->asset_ref().asset();
339 optional<string> pkl_hash;
340 BOOST_FOREACH (shared_ptr<PKL> i, pkls) {
341 pkl_hash = i->hash (reel_mxf->asset_ref()->id());
347 DCP_ASSERT (pkl_hash);
349 optional<string> cpl_hash = reel_mxf->hash();
350 if (cpl_hash && *cpl_hash != *pkl_hash) {
351 return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER;
354 if (actual_hash != *pkl_hash) {
355 return VERIFY_ASSET_RESULT_BAD;
358 return VERIFY_ASSET_RESULT_GOOD;
362 enum VerifyPictureAssetResult
364 VERIFY_PICTURE_ASSET_RESULT_GOOD,
365 VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG,
366 VERIFY_PICTURE_ASSET_RESULT_BAD,
371 biggest_frame_size (shared_ptr<const MonoPictureFrame> frame)
373 return frame->j2k_size ();
377 biggest_frame_size (shared_ptr<const StereoPictureFrame> frame)
379 return max(frame->left_j2k_size(), frame->right_j2k_size());
383 template <class A, class R, class F>
384 optional<VerifyPictureAssetResult>
385 verify_picture_asset_type (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
387 shared_ptr<A> asset = dynamic_pointer_cast<A>(reel_mxf->asset_ref().asset());
389 return optional<VerifyPictureAssetResult>();
392 int biggest_frame = 0;
393 shared_ptr<R> reader = asset->start_read ();
394 int64_t const duration = asset->intrinsic_duration ();
395 for (int64_t i = 0; i < duration; ++i) {
396 shared_ptr<const F> frame = reader->get_frame (i);
397 biggest_frame = max(biggest_frame, biggest_frame_size(frame));
398 progress (float(i) / duration);
401 static const int max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float()));
402 static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float()));
403 if (biggest_frame > max_frame) {
404 return VERIFY_PICTURE_ASSET_RESULT_BAD;
405 } else if (biggest_frame > risky_frame) {
406 return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG;
409 return VERIFY_PICTURE_ASSET_RESULT_GOOD;
413 static VerifyPictureAssetResult
414 verify_picture_asset (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
416 optional<VerifyPictureAssetResult> r = verify_picture_asset_type<MonoPictureAsset, MonoPictureAssetReader, MonoPictureFrame>(reel_mxf, progress);
418 r = verify_picture_asset_type<StereoPictureAsset, StereoPictureAssetReader, StereoPictureFrame>(reel_mxf, progress);
427 verify_main_picture_asset (
428 shared_ptr<const DCP> dcp,
429 shared_ptr<const Reel> reel,
430 function<void (string, optional<boost::filesystem::path>)> stage,
431 function<void (float)> progress,
432 list<VerificationNote>& notes
435 boost::filesystem::path const file = *reel->main_picture()->asset()->file();
436 stage ("Checking picture asset hash", file);
437 VerifyAssetResult const r = verify_asset (dcp, reel->main_picture(), progress);
439 case VERIFY_ASSET_RESULT_BAD:
442 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, file
446 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
449 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE, file
456 stage ("Checking picture frame sizes", reel->main_picture()->asset()->file());
457 VerifyPictureAssetResult const pr = verify_picture_asset (reel->main_picture(), progress);
459 case VERIFY_PICTURE_ASSET_RESULT_BAD:
462 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_FRAME_TOO_LARGE, file
466 case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG:
469 VerificationNote::VERIFY_WARNING, VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE, file
480 verify_main_sound_asset (
481 shared_ptr<const DCP> dcp,
482 shared_ptr<const Reel> reel,
483 function<void (string, optional<boost::filesystem::path>)> stage,
484 function<void (float)> progress,
485 list<VerificationNote>& notes
488 stage ("Checking sound asset hash", reel->main_sound()->asset()->file());
489 VerifyAssetResult const r = verify_asset (dcp, reel->main_sound(), progress);
491 case VERIFY_ASSET_RESULT_BAD:
494 VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *reel->main_sound()->asset()->file()
498 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
501 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE, *reel->main_sound()->asset()->file()
512 verify_main_subtitle_asset (
513 shared_ptr<const Reel> reel,
514 function<void (string, optional<boost::filesystem::path>)> stage,
515 boost::filesystem::path xsd_dtd_directory,
516 list<VerificationNote>& notes
519 shared_ptr<ReelSubtitleAsset> reel_asset = reel->main_subtitle ();
520 stage ("Checking subtitle XML", reel->main_subtitle()->asset()->file());
521 /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk
522 * gets passed through libdcp which may clean up and therefore hide errors.
524 validate_xml (reel->main_subtitle()->asset()->raw_xml(), xsd_dtd_directory, notes);
528 list<VerificationNote>
530 vector<boost::filesystem::path> directories,
531 function<void (string, optional<boost::filesystem::path>)> stage,
532 function<void (float)> progress,
533 boost::filesystem::path xsd_dtd_directory
536 xsd_dtd_directory = boost::filesystem::canonical (xsd_dtd_directory);
538 list<VerificationNote> notes;
540 list<shared_ptr<DCP> > dcps;
541 BOOST_FOREACH (boost::filesystem::path i, directories) {
542 dcps.push_back (shared_ptr<DCP> (new DCP (i)));
545 BOOST_FOREACH (shared_ptr<DCP> dcp, dcps) {
546 stage ("Checking DCP", dcp->directory());
549 } catch (ReadError& e) {
550 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
551 } catch (XMLError& e) {
552 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
555 BOOST_FOREACH (shared_ptr<CPL> cpl, dcp->cpls()) {
556 stage ("Checking CPL", cpl->file());
557 validate_xml (cpl->file().get(), xsd_dtd_directory, notes);
559 /* Check that the CPL's hash corresponds to the PKL */
560 BOOST_FOREACH (shared_ptr<PKL> i, dcp->pkls()) {
561 optional<string> h = i->hash(cpl->id());
562 if (h && make_digest(Data(*cpl->file())) != *h) {
563 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::CPL_HASH_INCORRECT));
567 BOOST_FOREACH (shared_ptr<Reel> reel, cpl->reels()) {
568 stage ("Checking reel", optional<boost::filesystem::path>());
570 BOOST_FOREACH (shared_ptr<ReelAsset> i, reel->assets()) {
571 if (i->duration() && (i->duration().get() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
572 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::DURATION_TOO_SMALL, i->id()));
574 if ((i->intrinsic_duration() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
575 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INTRINSIC_DURATION_TOO_SMALL, i->id()));
579 if (reel->main_picture()) {
580 /* Check reel stuff */
581 Fraction const frame_rate = reel->main_picture()->frame_rate();
582 if (frame_rate.denominator != 1 ||
583 (frame_rate.numerator != 24 &&
584 frame_rate.numerator != 25 &&
585 frame_rate.numerator != 30 &&
586 frame_rate.numerator != 48 &&
587 frame_rate.numerator != 50 &&
588 frame_rate.numerator != 60 &&
589 frame_rate.numerator != 96)) {
590 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INVALID_PICTURE_FRAME_RATE));
593 if (reel->main_picture()->asset_ref().resolved()) {
594 verify_main_picture_asset (dcp, reel, stage, progress, notes);
598 if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
599 verify_main_sound_asset (dcp, reel, stage, progress, notes);
602 if (reel->main_subtitle() && reel->main_subtitle()->asset_ref().resolved()) {
603 verify_main_subtitle_asset (reel, stage, xsd_dtd_directory, notes);
608 BOOST_FOREACH (shared_ptr<PKL> pkl, dcp->pkls()) {
609 stage ("Checking PKL", pkl->file());
610 validate_xml (pkl->file().get(), xsd_dtd_directory, notes);
613 if (dcp->asset_map_path()) {
614 stage ("Checking ASSETMAP", dcp->asset_map_path().get());
615 validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
617 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::MISSING_ASSETMAP));
625 dcp::note_to_string (dcp::VerificationNote note)
627 switch (note.code()) {
628 case dcp::VerificationNote::GENERAL_READ:
630 case dcp::VerificationNote::CPL_HASH_INCORRECT:
631 return "The hash of the CPL in the PKL does not agree with the CPL file.";
632 case dcp::VerificationNote::INVALID_PICTURE_FRAME_RATE:
633 return "The picture in a reel has an invalid frame rate.";
634 case dcp::VerificationNote::PICTURE_HASH_INCORRECT:
635 return dcp::String::compose("The hash of the picture asset %1 does not agree with the PKL file.", note.file()->filename());
636 case dcp::VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE:
637 return dcp::String::compose("The PKL and CPL hashes disagree for the picture asset %1.", note.file()->filename());
638 case dcp::VerificationNote::SOUND_HASH_INCORRECT:
639 return dcp::String::compose("The hash of the sound asset %1 does not agree with the PKL file.", note.file()->filename());
640 case dcp::VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE:
641 return dcp::String::compose("The PKL and CPL hashes disagree for the sound asset %1.", note.file()->filename());
642 case dcp::VerificationNote::EMPTY_ASSET_PATH:
643 return "The asset map contains an empty asset path.";
644 case dcp::VerificationNote::MISSING_ASSET:
645 return String::compose("The file for an asset in the asset map cannot be found; missing file is %1.", note.file()->filename());
646 case dcp::VerificationNote::MISMATCHED_STANDARD:
647 return "The DCP contains both SMPTE and Interop parts.";
648 case dcp::VerificationNote::XML_VALIDATION_ERROR:
649 return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get());
650 case dcp::VerificationNote::MISSING_ASSETMAP:
651 return "No ASSETMAP or ASSETMAP.xml was found.";
652 case dcp::VerificationNote::INTRINSIC_DURATION_TOO_SMALL:
653 return String::compose("The intrinsic duration of an asset is less than 1 second long: %1", note.note().get());
654 case dcp::VerificationNote::DURATION_TOO_SMALL:
655 return String::compose("The duration of an asset is less than 1 second long: %1", note.note().get());
656 case dcp::VerificationNote::PICTURE_FRAME_TOO_LARGE:
657 return String::compose("The instantaneous bit rate of the picture asset %1 is larger than the limit of 250Mbit/s in at least one place.", note.file()->filename());
658 case dcp::VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE:
659 return String::compose("The instantaneous bit rate of the picture asset %1 is close to the limit of 250Mbit/s in at least one place.", note.file()->filename());
660 case dcp::VerificationNote::EXTERNAL_ASSET:
661 return String::compose("An asset that this DCP refers to is not included in the DCP. It may be a VF. Missing asset is %1.", note.note().get());