2 Copyright (C) 2018-2020 Carl Hetherington <cth@carlh.net>
4 This file is part of libdcp.
6 libdcp is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 libdcp is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with libdcp. If not, see <http://www.gnu.org/licenses/>.
19 In addition, as a special exception, the copyright holders give
20 permission to link the code of portions of this program with the
21 OpenSSL library under certain conditions as described in each
22 individual source file, and distribute linked combinations
25 You must obey the GNU General Public License in all respects
26 for all of the code used other than OpenSSL. If you modify
27 file(s) with this exception, you may extend this exception to your
28 version of the file(s), but you are not obligated to do so. If you
29 do not wish to do so, delete this exception statement from your
30 version. If you delete this exception statement from all source
31 files in the program, then also delete it here.
38 #include "reel_picture_asset.h"
39 #include "reel_sound_asset.h"
40 #include "reel_subtitle_asset.h"
41 #include "interop_subtitle_asset.h"
42 #include "mono_picture_asset.h"
43 #include "mono_picture_frame.h"
44 #include "stereo_picture_asset.h"
45 #include "stereo_picture_frame.h"
46 #include "exceptions.h"
47 #include "compose.hpp"
48 #include "raw_convert.h"
49 #include <xercesc/util/PlatformUtils.hpp>
50 #include <xercesc/parsers/XercesDOMParser.hpp>
51 #include <xercesc/parsers/AbstractDOMParser.hpp>
52 #include <xercesc/sax/HandlerBase.hpp>
53 #include <xercesc/dom/DOMImplementation.hpp>
54 #include <xercesc/dom/DOMImplementationLS.hpp>
55 #include <xercesc/dom/DOMImplementationRegistry.hpp>
56 #include <xercesc/dom/DOMLSParser.hpp>
57 #include <xercesc/dom/DOMException.hpp>
58 #include <xercesc/dom/DOMDocument.hpp>
59 #include <xercesc/dom/DOMNodeList.hpp>
60 #include <xercesc/dom/DOMError.hpp>
61 #include <xercesc/dom/DOMLocator.hpp>
62 #include <xercesc/dom/DOMNamedNodeMap.hpp>
63 #include <xercesc/dom/DOMAttr.hpp>
64 #include <xercesc/dom/DOMErrorHandler.hpp>
65 #include <xercesc/framework/LocalFileInputSource.hpp>
66 #include <xercesc/framework/MemBufInputSource.hpp>
67 #include <boost/noncopyable.hpp>
68 #include <boost/foreach.hpp>
69 #include <boost/algorithm/string.hpp>
81 using boost::shared_ptr;
82 using boost::optional;
83 using boost::function;
84 using boost::dynamic_pointer_cast;
87 using namespace xercesc;
91 xml_ch_to_string (XMLCh const * a)
93 char* x = XMLString::transcode(a);
95 XMLString::release(&x);
99 class XMLValidationError
102 XMLValidationError (SAXParseException const & e)
103 : _message (xml_ch_to_string(e.getMessage()))
104 , _line (e.getLineNumber())
105 , _column (e.getColumnNumber())
110 string message () const {
114 uint64_t line () const {
118 uint64_t column () const {
129 class DCPErrorHandler : public ErrorHandler
132 void warning(const SAXParseException& e)
134 maybe_add (XMLValidationError(e));
137 void error(const SAXParseException& e)
139 maybe_add (XMLValidationError(e));
142 void fatalError(const SAXParseException& e)
144 maybe_add (XMLValidationError(e));
151 list<XMLValidationError> errors () const {
156 void maybe_add (XMLValidationError e)
158 /* XXX: nasty hack */
160 e.message().find("schema document") != string::npos &&
161 e.message().find("has different target namespace from the one specified in instance document") != string::npos
166 _errors.push_back (e);
169 list<XMLValidationError> _errors;
172 class StringToXMLCh : public boost::noncopyable
175 StringToXMLCh (string a)
177 _buffer = XMLString::transcode(a.c_str());
182 XMLString::release (&_buffer);
185 XMLCh const * get () const {
193 class LocalFileResolver : public EntityResolver
196 LocalFileResolver (boost::filesystem::path xsd_dtd_directory)
197 : _xsd_dtd_directory (xsd_dtd_directory)
199 add("http://www.w3.org/2001/XMLSchema.dtd", "XMLSchema.dtd");
200 add("http://www.w3.org/2001/03/xml.xsd", "xml.xsd");
201 add("http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd", "xmldsig-core-schema.xsd");
202 add("http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd", "Main-Stereo-Picture-CPL.xsd");
203 add("http://www.digicine.com/PROTO-ASDCP-CPL-20040511.xsd", "PROTO-ASDCP-CPL-20040511.xsd");
204 add("http://www.digicine.com/PROTO-ASDCP-PKL-20040311.xsd", "PROTO-ASDCP-PKL-20040311.xsd");
205 add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd");
206 add("interop-subs", "DCSubtitle.v1.mattsson.xsd");
207 add("http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd", "SMPTE-428-7-2010-DCST.xsd");
210 InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id)
212 string system_id_str = xml_ch_to_string (system_id);
213 if (_files.find(system_id_str) == _files.end()) {
217 boost::filesystem::path p = _xsd_dtd_directory / _files[system_id_str];
218 StringToXMLCh ch (p.string());
219 return new LocalFileInputSource(ch.get());
223 void add (string uri, string file)
228 std::map<string, string> _files;
229 boost::filesystem::path _xsd_dtd_directory;
234 parse (XercesDOMParser& parser, boost::filesystem::path xml)
236 parser.parse(xml.string().c_str());
241 parse (XercesDOMParser& parser, std::string xml)
243 xercesc::MemBufInputSource buf(reinterpret_cast<unsigned char const*>(xml.c_str()), xml.size(), "");
250 validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, list<VerificationNote>& notes)
253 XMLPlatformUtils::Initialize ();
254 } catch (XMLException& e) {
255 throw MiscError ("Failed to initialise xerces library");
258 DCPErrorHandler error_handler;
260 /* All the xerces objects in this scope must be destroyed before XMLPlatformUtils::Terminate() is called */
262 XercesDOMParser parser;
263 parser.setValidationScheme(XercesDOMParser::Val_Always);
264 parser.setDoNamespaces(true);
265 parser.setDoSchema(true);
267 map<string, string> schema;
268 schema["http://www.w3.org/2000/09/xmldsig#"] = "xmldsig-core-schema.xsd";
269 schema["http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd"] = "xmldsig-core-schema.xsd";
270 schema["http://www.smpte-ra.org/schemas/429-7/2006/CPL"] = "SMPTE-429-7-2006-CPL.xsd";
271 schema["http://www.smpte-ra.org/schemas/429-8/2006/PKL"] = "SMPTE-429-8-2006-PKL.xsd";
272 schema["http://www.smpte-ra.org/schemas/429-9/2007/AM"] = "SMPTE-429-9-2007-AM.xsd";
273 schema["http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd"] = "Main-Stereo-Picture-CPL.xsd";
274 schema["http://www.digicine.com/PROTO-ASDCP-CPL-20040511#"] = "PROTO-ASDCP-CPL-20040511.xsd";
275 schema["http://www.digicine.com/PROTO-ASDCP-PKL-20040311#"] = "PROTO-ASDCP-PKL-20040311.xsd";
276 schema["http://www.digicine.com/PROTO-ASDCP-AM-20040311#"] = "PROTO-ASDCP-AM-20040311.xsd";
277 schema["interop-subs"] = "DCSubtitle.v1.mattsson.xsd";
278 schema["http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd"] = "DCDMSubtitle-2010.xsd";
281 for (map<string, string>::const_iterator i = schema.begin(); i != schema.end(); ++i) {
282 locations += i->first;
284 boost::filesystem::path p = xsd_dtd_directory / i->second;
285 locations += p.string() + " ";
288 parser.setExternalSchemaLocation(locations.c_str());
289 parser.setValidationSchemaFullChecking(true);
290 parser.setErrorHandler(&error_handler);
292 LocalFileResolver resolver (xsd_dtd_directory);
293 parser.setEntityResolver(&resolver);
296 parser.resetDocumentPool();
298 } catch (XMLException& e) {
299 throw MiscError(xml_ch_to_string(e.getMessage()));
300 } catch (DOMException& e) {
301 throw MiscError(xml_ch_to_string(e.getMessage()));
303 throw MiscError("Unknown exception from xerces");
307 XMLPlatformUtils::Terminate ();
309 BOOST_FOREACH (XMLValidationError i, error_handler.errors()) {
312 VerificationNote::VERIFY_ERROR,
313 VerificationNote::XML_VALIDATION_ERROR,
323 enum VerifyAssetResult {
324 VERIFY_ASSET_RESULT_GOOD,
325 VERIFY_ASSET_RESULT_CPL_PKL_DIFFER,
326 VERIFY_ASSET_RESULT_BAD
330 static VerifyAssetResult
331 verify_asset (shared_ptr<const DCP> dcp, shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
333 string const actual_hash = reel_mxf->asset_ref()->hash(progress);
335 list<shared_ptr<PKL> > pkls = dcp->pkls();
336 /* We've read this DCP in so it must have at least one PKL */
337 DCP_ASSERT (!pkls.empty());
339 shared_ptr<Asset> asset = reel_mxf->asset_ref().asset();
341 optional<string> pkl_hash;
342 BOOST_FOREACH (shared_ptr<PKL> i, pkls) {
343 pkl_hash = i->hash (reel_mxf->asset_ref()->id());
349 DCP_ASSERT (pkl_hash);
351 optional<string> cpl_hash = reel_mxf->hash();
352 if (cpl_hash && *cpl_hash != *pkl_hash) {
353 return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER;
356 if (actual_hash != *pkl_hash) {
357 return VERIFY_ASSET_RESULT_BAD;
360 return VERIFY_ASSET_RESULT_GOOD;
364 enum VerifyPictureAssetResult
366 VERIFY_PICTURE_ASSET_RESULT_GOOD,
367 VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG,
368 VERIFY_PICTURE_ASSET_RESULT_BAD,
373 biggest_frame_size (shared_ptr<const MonoPictureFrame> frame)
375 return frame->j2k_size ();
379 biggest_frame_size (shared_ptr<const StereoPictureFrame> frame)
381 return max(frame->left_j2k_size(), frame->right_j2k_size());
385 template <class A, class R, class F>
386 optional<VerifyPictureAssetResult>
387 verify_picture_asset_type (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
389 shared_ptr<A> asset = dynamic_pointer_cast<A>(reel_mxf->asset_ref().asset());
391 return optional<VerifyPictureAssetResult>();
394 int biggest_frame = 0;
395 shared_ptr<R> reader = asset->start_read ();
396 int64_t const duration = asset->intrinsic_duration ();
397 for (int64_t i = 0; i < duration; ++i) {
398 shared_ptr<const F> frame = reader->get_frame (i);
399 biggest_frame = max(biggest_frame, biggest_frame_size(frame));
400 progress (float(i) / duration);
403 static const int max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float()));
404 static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float()));
405 if (biggest_frame > max_frame) {
406 return VERIFY_PICTURE_ASSET_RESULT_BAD;
407 } else if (biggest_frame > risky_frame) {
408 return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG;
411 return VERIFY_PICTURE_ASSET_RESULT_GOOD;
415 static VerifyPictureAssetResult
416 verify_picture_asset (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
418 optional<VerifyPictureAssetResult> r = verify_picture_asset_type<MonoPictureAsset, MonoPictureAssetReader, MonoPictureFrame>(reel_mxf, progress);
420 r = verify_picture_asset_type<StereoPictureAsset, StereoPictureAssetReader, StereoPictureFrame>(reel_mxf, progress);
429 verify_main_picture_asset (
430 shared_ptr<const DCP> dcp,
431 shared_ptr<const Reel> reel,
432 function<void (string, optional<boost::filesystem::path>)> stage,
433 function<void (float)> progress,
434 list<VerificationNote>& notes
437 boost::filesystem::path const file = *reel->main_picture()->asset()->file();
438 stage ("Checking picture asset hash", file);
439 VerifyAssetResult const r = verify_asset (dcp, reel->main_picture(), progress);
441 case VERIFY_ASSET_RESULT_BAD:
444 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, file
448 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
451 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE, file
458 stage ("Checking picture frame sizes", reel->main_picture()->asset()->file());
459 VerifyPictureAssetResult const pr = verify_picture_asset (reel->main_picture(), progress);
461 case VERIFY_PICTURE_ASSET_RESULT_BAD:
464 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_FRAME_TOO_LARGE, file
468 case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG:
471 VerificationNote::VERIFY_WARNING, VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE, file
482 verify_main_sound_asset (
483 shared_ptr<const DCP> dcp,
484 shared_ptr<const Reel> reel,
485 function<void (string, optional<boost::filesystem::path>)> stage,
486 function<void (float)> progress,
487 list<VerificationNote>& notes
490 stage ("Checking sound asset hash", reel->main_sound()->asset()->file());
491 VerifyAssetResult const r = verify_asset (dcp, reel->main_sound(), progress);
493 case VERIFY_ASSET_RESULT_BAD:
496 VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *reel->main_sound()->asset()->file()
500 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
503 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE, *reel->main_sound()->asset()->file()
514 verify_main_subtitle_asset (
515 shared_ptr<const Reel> reel,
516 function<void (string, optional<boost::filesystem::path>)> stage,
517 boost::filesystem::path xsd_dtd_directory,
518 list<VerificationNote>& notes
521 shared_ptr<ReelSubtitleAsset> reel_asset = reel->main_subtitle ();
522 stage ("Checking subtitle XML", reel->main_subtitle()->asset()->file());
523 /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk
524 * gets passed through libdcp which may clean up and therefore hide errors.
526 validate_xml (reel->main_subtitle()->asset()->raw_xml(), xsd_dtd_directory, notes);
530 list<VerificationNote>
532 vector<boost::filesystem::path> directories,
533 function<void (string, optional<boost::filesystem::path>)> stage,
534 function<void (float)> progress,
535 boost::filesystem::path xsd_dtd_directory
538 xsd_dtd_directory = boost::filesystem::canonical (xsd_dtd_directory);
540 list<VerificationNote> notes;
542 list<shared_ptr<DCP> > dcps;
543 BOOST_FOREACH (boost::filesystem::path i, directories) {
544 dcps.push_back (shared_ptr<DCP> (new DCP (i)));
547 BOOST_FOREACH (shared_ptr<DCP> dcp, dcps) {
548 stage ("Checking DCP", dcp->directory());
551 } catch (ReadError& e) {
552 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
553 } catch (XMLError& e) {
554 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
557 BOOST_FOREACH (shared_ptr<CPL> cpl, dcp->cpls()) {
558 stage ("Checking CPL", cpl->file());
559 validate_xml (cpl->file().get(), xsd_dtd_directory, notes);
561 /* Check that the CPL's hash corresponds to the PKL */
562 BOOST_FOREACH (shared_ptr<PKL> i, dcp->pkls()) {
563 optional<string> h = i->hash(cpl->id());
564 if (h && make_digest(Data(*cpl->file())) != *h) {
565 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::CPL_HASH_INCORRECT));
569 BOOST_FOREACH (shared_ptr<Reel> reel, cpl->reels()) {
570 stage ("Checking reel", optional<boost::filesystem::path>());
572 BOOST_FOREACH (shared_ptr<ReelAsset> i, reel->assets()) {
573 if (i->duration() && (i->duration().get() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
574 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::DURATION_TOO_SMALL, i->id()));
576 if ((i->intrinsic_duration() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
577 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INTRINSIC_DURATION_TOO_SMALL, i->id()));
581 if (reel->main_picture()) {
582 /* Check reel stuff */
583 Fraction const frame_rate = reel->main_picture()->frame_rate();
584 if (frame_rate.denominator != 1 ||
585 (frame_rate.numerator != 24 &&
586 frame_rate.numerator != 25 &&
587 frame_rate.numerator != 30 &&
588 frame_rate.numerator != 48 &&
589 frame_rate.numerator != 50 &&
590 frame_rate.numerator != 60 &&
591 frame_rate.numerator != 96)) {
592 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INVALID_PICTURE_FRAME_RATE));
595 if (reel->main_picture()->asset_ref().resolved()) {
596 verify_main_picture_asset (dcp, reel, stage, progress, notes);
600 if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
601 verify_main_sound_asset (dcp, reel, stage, progress, notes);
604 if (reel->main_subtitle() && reel->main_subtitle()->asset_ref().resolved()) {
605 verify_main_subtitle_asset (reel, stage, xsd_dtd_directory, notes);
610 BOOST_FOREACH (shared_ptr<PKL> pkl, dcp->pkls()) {
611 stage ("Checking PKL", pkl->file());
612 validate_xml (pkl->file().get(), xsd_dtd_directory, notes);
615 if (dcp->asset_map_path()) {
616 stage ("Checking ASSETMAP", dcp->asset_map_path().get());
617 validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
619 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::MISSING_ASSETMAP));
627 dcp::note_to_string (dcp::VerificationNote note)
629 switch (note.code()) {
630 case dcp::VerificationNote::GENERAL_READ:
632 case dcp::VerificationNote::CPL_HASH_INCORRECT:
633 return "The hash of the CPL in the PKL does not agree with the CPL file";
634 case dcp::VerificationNote::INVALID_PICTURE_FRAME_RATE:
635 return "The picture in a reel has an invalid frame rate";
636 case dcp::VerificationNote::PICTURE_HASH_INCORRECT:
637 return dcp::String::compose("The hash of the picture asset %1 does not agree with the PKL file", note.file()->filename());
638 case dcp::VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE:
639 return dcp::String::compose("The PKL and CPL hashes disagree for the picture asset %1", note.file()->filename());
640 case dcp::VerificationNote::SOUND_HASH_INCORRECT:
641 return dcp::String::compose("The hash of the sound asset %1 does not agree with the PKL file", note.file()->filename());
642 case dcp::VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE:
643 return dcp::String::compose("The PKL and CPL hashes disagree for the sound asset %1", note.file()->filename());
644 case dcp::VerificationNote::EMPTY_ASSET_PATH:
645 return "The asset map contains an empty asset path.";
646 case dcp::VerificationNote::MISSING_ASSET:
647 return String::compose("The file for an asset in the asset map cannot be found; missing file is %1.", note.file()->filename());
648 case dcp::VerificationNote::MISMATCHED_STANDARD:
649 return "The DCP contains both SMPTE and Interop parts.";
650 case dcp::VerificationNote::XML_VALIDATION_ERROR:
651 return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get());
652 case dcp::VerificationNote::MISSING_ASSETMAP:
653 return "No ASSETMAP or ASSETMAP.xml was found";
654 case dcp::VerificationNote::INTRINSIC_DURATION_TOO_SMALL:
655 return String::compose("The intrinsic duration of an asset is less than 1 second long: %1", note.note().get());
656 case dcp::VerificationNote::DURATION_TOO_SMALL:
657 return String::compose("The duration of an asset is less than 1 second long: %1", note.note().get());
658 case dcp::VerificationNote::PICTURE_FRAME_TOO_LARGE:
659 return String::compose("The instantaneous bit rate of the picture asset %1 is larger than the limit of 250Mbit/s in at least one place", note.file()->filename());
660 case dcp::VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE:
661 return String::compose("The instantaneous bit rate of the picture asset %1 is close to the limit of 250Mbit/s in at least one place", note.file()->filename());
662 case dcp::VerificationNote::EXTERNAL_ASSET:
663 return "An asset that this DCP refers to is not included in the DCP. It may be a VF.";