2 Copyright (C) 2018-2020 Carl Hetherington <cth@carlh.net>
4 This file is part of libdcp.
6 libdcp is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 libdcp is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with libdcp. If not, see <http://www.gnu.org/licenses/>.
19 In addition, as a special exception, the copyright holders give
20 permission to link the code of portions of this program with the
21 OpenSSL library under certain conditions as described in each
22 individual source file, and distribute linked combinations
25 You must obey the GNU General Public License in all respects
26 for all of the code used other than OpenSSL. If you modify
27 file(s) with this exception, you may extend this exception to your
28 version of the file(s), but you are not obligated to do so. If you
29 do not wish to do so, delete this exception statement from your
30 version. If you delete this exception statement from all source
31 files in the program, then also delete it here.
38 #include "reel_picture_asset.h"
39 #include "reel_sound_asset.h"
40 #include "mono_picture_asset.h"
41 #include "mono_picture_frame.h"
42 #include "stereo_picture_asset.h"
43 #include "stereo_picture_frame.h"
44 #include "exceptions.h"
45 #include "compose.hpp"
46 #include "raw_convert.h"
47 #include <xercesc/util/PlatformUtils.hpp>
48 #include <xercesc/parsers/XercesDOMParser.hpp>
49 #include <xercesc/parsers/AbstractDOMParser.hpp>
50 #include <xercesc/sax/HandlerBase.hpp>
51 #include <xercesc/dom/DOMImplementation.hpp>
52 #include <xercesc/dom/DOMImplementationLS.hpp>
53 #include <xercesc/dom/DOMImplementationRegistry.hpp>
54 #include <xercesc/dom/DOMLSParser.hpp>
55 #include <xercesc/dom/DOMException.hpp>
56 #include <xercesc/dom/DOMDocument.hpp>
57 #include <xercesc/dom/DOMNodeList.hpp>
58 #include <xercesc/dom/DOMError.hpp>
59 #include <xercesc/dom/DOMLocator.hpp>
60 #include <xercesc/dom/DOMNamedNodeMap.hpp>
61 #include <xercesc/dom/DOMAttr.hpp>
62 #include <xercesc/dom/DOMErrorHandler.hpp>
63 #include <xercesc/framework/LocalFileInputSource.hpp>
64 #include <boost/noncopyable.hpp>
65 #include <boost/foreach.hpp>
66 #include <boost/algorithm/string.hpp>
78 using boost::shared_ptr;
79 using boost::optional;
80 using boost::function;
81 using boost::dynamic_pointer_cast;
84 using namespace xercesc;
88 xml_ch_to_string (XMLCh const * a)
90 char* x = XMLString::transcode(a);
92 XMLString::release(&x);
96 class XMLValidationError
99 XMLValidationError (SAXParseException const & e)
100 : _message (xml_ch_to_string(e.getMessage()))
101 , _line (e.getLineNumber())
102 , _column (e.getColumnNumber())
107 string message () const {
111 uint64_t line () const {
115 uint64_t column () const {
126 class DCPErrorHandler : public ErrorHandler
129 void warning(const SAXParseException& e)
131 maybe_add (XMLValidationError(e));
134 void error(const SAXParseException& e)
136 maybe_add (XMLValidationError(e));
139 void fatalError(const SAXParseException& e)
141 maybe_add (XMLValidationError(e));
148 list<XMLValidationError> errors () const {
153 void maybe_add (XMLValidationError e)
155 /* XXX: nasty hack */
157 e.message().find("schema document") != string::npos &&
158 e.message().find("has different target namespace from the one specified in instance document") != string::npos
163 _errors.push_back (e);
166 list<XMLValidationError> _errors;
169 class StringToXMLCh : public boost::noncopyable
172 StringToXMLCh (string a)
174 _buffer = XMLString::transcode(a.c_str());
179 XMLString::release (&_buffer);
182 XMLCh const * get () const {
190 class LocalFileResolver : public EntityResolver
193 LocalFileResolver (boost::filesystem::path xsd_dtd_directory)
194 : _xsd_dtd_directory (xsd_dtd_directory)
196 add("http://www.w3.org/2001/XMLSchema.dtd", "XMLSchema.dtd");
197 add("http://www.w3.org/2001/03/xml.xsd", "xml.xsd");
198 add("http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd", "xmldsig-core-schema.xsd");
199 add("http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd", "Main-Stereo-Picture-CPL.xsd");
200 add("http://www.digicine.com/PROTO-ASDCP-CPL-20040511.xsd", "PROTO-ASDCP-CPL-20040511.xsd");
201 add("http://www.digicine.com/PROTO-ASDCP-PKL-20040311.xsd", "PROTO-ASDCP-PKL-20040311.xsd");
202 add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd");
205 InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id)
207 string system_id_str = xml_ch_to_string (system_id);
208 if (_files.find(system_id_str) == _files.end()) {
212 boost::filesystem::path p = _xsd_dtd_directory / _files[system_id_str];
213 StringToXMLCh ch (p.string());
214 return new LocalFileInputSource(ch.get());
218 void add (string uri, string file)
223 std::map<string, string> _files;
224 boost::filesystem::path _xsd_dtd_directory;
229 validate_xml (boost::filesystem::path xml_file, boost::filesystem::path xsd_dtd_directory, list<VerificationNote>& notes)
232 XMLPlatformUtils::Initialize ();
233 } catch (XMLException& e) {
234 throw MiscError ("Failed to initialise xerces library");
237 DCPErrorHandler error_handler;
239 /* All the xerces objects in this scope must be destroyed before XMLPlatformUtils::Terminate() is called */
241 XercesDOMParser parser;
242 parser.setValidationScheme(XercesDOMParser::Val_Always);
243 parser.setDoNamespaces(true);
244 parser.setDoSchema(true);
246 map<string, string> schema;
247 schema["http://www.w3.org/2000/09/xmldsig#"] = "xmldsig-core-schema.xsd";
248 schema["http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd"] = "xmldsig-core-schema.xsd";
249 schema["http://www.smpte-ra.org/schemas/429-7/2006/CPL"] = "SMPTE-429-7-2006-CPL.xsd";
250 schema["http://www.smpte-ra.org/schemas/429-8/2006/PKL"] = "SMPTE-429-8-2006-PKL.xsd";
251 schema["http://www.smpte-ra.org/schemas/429-9/2007/AM"] = "SMPTE-429-9-2007-AM.xsd";
252 schema["http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd"] = "Main-Stereo-Picture-CPL.xsd";
253 schema["http://www.digicine.com/PROTO-ASDCP-CPL-20040511#"] = "PROTO-ASDCP-CPL-20040511.xsd";
254 schema["http://www.digicine.com/PROTO-ASDCP-PKL-20040311#"] = "PROTO-ASDCP-PKL-20040311.xsd";
255 schema["http://www.digicine.com/PROTO-ASDCP-AM-20040311#"] = "PROTO-ASDCP-AM-20040311.xsd";
258 for (map<string, string>::const_iterator i = schema.begin(); i != schema.end(); ++i) {
259 locations += i->first;
261 boost::filesystem::path p = xsd_dtd_directory / i->second;
262 locations += p.string() + " ";
265 parser.setExternalSchemaLocation(locations.c_str());
266 parser.setValidationSchemaFullChecking(true);
267 parser.setErrorHandler(&error_handler);
269 LocalFileResolver resolver (xsd_dtd_directory);
270 parser.setEntityResolver(&resolver);
273 parser.resetDocumentPool();
274 parser.parse(xml_file.string().c_str());
275 } catch (XMLException& e) {
276 throw MiscError(xml_ch_to_string(e.getMessage()));
277 } catch (DOMException& e) {
278 throw MiscError(xml_ch_to_string(e.getMessage()));
280 throw MiscError("Unknown exception from xerces");
284 XMLPlatformUtils::Terminate ();
286 BOOST_FOREACH (XMLValidationError i, error_handler.errors()) {
289 VerificationNote::VERIFY_ERROR,
290 VerificationNote::XML_VALIDATION_ERROR,
300 enum VerifyAssetResult {
301 VERIFY_ASSET_RESULT_GOOD,
302 VERIFY_ASSET_RESULT_CPL_PKL_DIFFER,
303 VERIFY_ASSET_RESULT_BAD
307 static VerifyAssetResult
308 verify_asset (shared_ptr<DCP> dcp, shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
310 string const actual_hash = reel_mxf->asset_ref()->hash(progress);
312 list<shared_ptr<PKL> > pkls = dcp->pkls();
313 /* We've read this DCP in so it must have at least one PKL */
314 DCP_ASSERT (!pkls.empty());
316 shared_ptr<Asset> asset = reel_mxf->asset_ref().asset();
318 optional<string> pkl_hash;
319 BOOST_FOREACH (shared_ptr<PKL> i, pkls) {
320 pkl_hash = i->hash (reel_mxf->asset_ref()->id());
326 DCP_ASSERT (pkl_hash);
328 optional<string> cpl_hash = reel_mxf->hash();
329 if (cpl_hash && *cpl_hash != *pkl_hash) {
330 return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER;
333 if (actual_hash != *pkl_hash) {
334 return VERIFY_ASSET_RESULT_BAD;
337 return VERIFY_ASSET_RESULT_GOOD;
341 enum VerifyPictureAssetResult
343 VERIFY_PICTURE_ASSET_RESULT_GOOD,
344 VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG,
345 VERIFY_PICTURE_ASSET_RESULT_BAD,
350 biggest_frame_size (shared_ptr<const MonoPictureFrame> frame)
352 return frame->j2k_size ();
356 biggest_frame_size (shared_ptr<const StereoPictureFrame> frame)
358 return max(frame->left_j2k_size(), frame->right_j2k_size());
362 template <class A, class R, class F>
363 optional<VerifyPictureAssetResult>
364 verify_picture_asset_type (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
366 shared_ptr<A> asset = dynamic_pointer_cast<A>(reel_mxf->asset_ref().asset());
368 return optional<VerifyPictureAssetResult>();
371 int biggest_frame = 0;
372 shared_ptr<R> reader = asset->start_read ();
373 int64_t const duration = asset->intrinsic_duration ();
374 for (int64_t i = 0; i < duration; ++i) {
375 shared_ptr<const F> frame = reader->get_frame (i);
376 biggest_frame = max(biggest_frame, biggest_frame_size(frame));
377 progress (float(i) / duration);
380 static const int max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float()));
381 static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float()));
382 if (biggest_frame > max_frame) {
383 return VERIFY_PICTURE_ASSET_RESULT_BAD;
384 } else if (biggest_frame > risky_frame) {
385 return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG;
388 return VERIFY_PICTURE_ASSET_RESULT_GOOD;
392 static VerifyPictureAssetResult
393 verify_picture_asset (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
395 optional<VerifyPictureAssetResult> r = verify_picture_asset_type<MonoPictureAsset, MonoPictureAssetReader, MonoPictureFrame>(reel_mxf, progress);
397 r = verify_picture_asset_type<StereoPictureAsset, StereoPictureAssetReader, StereoPictureFrame>(reel_mxf, progress);
405 list<VerificationNote>
407 vector<boost::filesystem::path> directories,
408 function<void (string, optional<boost::filesystem::path>)> stage,
409 function<void (float)> progress,
410 boost::filesystem::path xsd_dtd_directory
413 xsd_dtd_directory = boost::filesystem::canonical (xsd_dtd_directory);
415 list<VerificationNote> notes;
417 list<shared_ptr<DCP> > dcps;
418 BOOST_FOREACH (boost::filesystem::path i, directories) {
419 dcps.push_back (shared_ptr<DCP> (new DCP (i)));
422 BOOST_FOREACH (shared_ptr<DCP> dcp, dcps) {
423 stage ("Checking DCP", dcp->directory());
426 } catch (ReadError& e) {
427 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
428 } catch (XMLError& e) {
429 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
432 BOOST_FOREACH (shared_ptr<CPL> cpl, dcp->cpls()) {
433 stage ("Checking CPL", cpl->file());
434 validate_xml (cpl->file().get(), xsd_dtd_directory, notes);
436 /* Check that the CPL's hash corresponds to the PKL */
437 BOOST_FOREACH (shared_ptr<PKL> i, dcp->pkls()) {
438 optional<string> h = i->hash(cpl->id());
439 if (h && make_digest(Data(*cpl->file())) != *h) {
440 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::CPL_HASH_INCORRECT));
444 BOOST_FOREACH (shared_ptr<Reel> reel, cpl->reels()) {
445 stage ("Checking reel", optional<boost::filesystem::path>());
447 BOOST_FOREACH (shared_ptr<ReelAsset> i, reel->assets()) {
448 if (i->duration() && (i->duration().get() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
449 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::DURATION_TOO_SMALL, i->id()));
451 if ((i->intrinsic_duration() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
452 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INTRINSIC_DURATION_TOO_SMALL, i->id()));
456 if (reel->main_picture()) {
457 /* Check reel stuff */
458 Fraction const frame_rate = reel->main_picture()->frame_rate();
459 if (frame_rate.denominator != 1 ||
460 (frame_rate.numerator != 24 &&
461 frame_rate.numerator != 25 &&
462 frame_rate.numerator != 30 &&
463 frame_rate.numerator != 48 &&
464 frame_rate.numerator != 50 &&
465 frame_rate.numerator != 60 &&
466 frame_rate.numerator != 96)) {
467 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INVALID_PICTURE_FRAME_RATE));
470 if (reel->main_picture()->asset_ref().resolved()) {
471 boost::filesystem::path const file = *reel->main_picture()->asset()->file();
472 stage ("Checking picture asset hash", file);
473 VerifyAssetResult const r = verify_asset (dcp, reel->main_picture(), progress);
475 case VERIFY_ASSET_RESULT_BAD:
478 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, file
482 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
485 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE, file
492 stage ("Checking picture frame sizes", reel->main_picture()->asset()->file());
493 VerifyPictureAssetResult const pr = verify_picture_asset (reel->main_picture(), progress);
495 case VERIFY_PICTURE_ASSET_RESULT_BAD:
498 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_FRAME_TOO_LARGE, file
502 case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG:
505 VerificationNote::VERIFY_WARNING, VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE, file
514 if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
515 stage ("Checking sound asset hash", reel->main_sound()->asset()->file());
516 VerifyAssetResult const r = verify_asset (dcp, reel->main_sound(), progress);
518 case VERIFY_ASSET_RESULT_BAD:
521 VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *reel->main_sound()->asset()->file()
525 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
528 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE, *reel->main_sound()->asset()->file()
539 BOOST_FOREACH (shared_ptr<PKL> pkl, dcp->pkls()) {
540 stage ("Checking PKL", pkl->file());
541 validate_xml (pkl->file().get(), xsd_dtd_directory, notes);
544 if (dcp->asset_map_path()) {
545 stage ("Checking ASSETMAP", dcp->asset_map_path().get());
546 validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
548 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::MISSING_ASSETMAP));
556 dcp::note_to_string (dcp::VerificationNote note)
558 switch (note.code()) {
559 case dcp::VerificationNote::GENERAL_READ:
561 case dcp::VerificationNote::CPL_HASH_INCORRECT:
562 return "The hash of the CPL in the PKL does not agree with the CPL file";
563 case dcp::VerificationNote::INVALID_PICTURE_FRAME_RATE:
564 return "The picture in a reel has an invalid frame rate";
565 case dcp::VerificationNote::PICTURE_HASH_INCORRECT:
566 return dcp::String::compose("The hash of the picture asset %1 does not agree with the PKL file", note.file()->filename());
567 case dcp::VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE:
568 return dcp::String::compose("The PKL and CPL hashes disagree for the picture asset %1", note.file()->filename());
569 case dcp::VerificationNote::SOUND_HASH_INCORRECT:
570 return dcp::String::compose("The hash of the sound asset %1 does not agree with the PKL file", note.file()->filename());
571 case dcp::VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE:
572 return dcp::String::compose("The PKL and CPL hashes disagree for the sound asset %1", note.file()->filename());
573 case dcp::VerificationNote::EMPTY_ASSET_PATH:
574 return "The asset map contains an empty asset path.";
575 case dcp::VerificationNote::MISSING_ASSET:
576 return String::compose("The file for an asset in the asset map cannot be found; missing file is %1.", note.file()->filename());
577 case dcp::VerificationNote::MISMATCHED_STANDARD:
578 return "The DCP contains both SMPTE and Interop parts.";
579 case dcp::VerificationNote::XML_VALIDATION_ERROR:
580 return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get());
581 case dcp::VerificationNote::MISSING_ASSETMAP:
582 return "No ASSETMAP or ASSETMAP.xml was found";
583 case dcp::VerificationNote::INTRINSIC_DURATION_TOO_SMALL:
584 return String::compose("The intrinsic duration of an asset is less than 1 second long: %1", note.note().get());
585 case dcp::VerificationNote::DURATION_TOO_SMALL:
586 return String::compose("The duration of an asset is less than 1 second long: %1", note.note().get());
587 case dcp::VerificationNote::PICTURE_FRAME_TOO_LARGE:
588 return String::compose("The instantaneous bit rate of the picture asset %1 is larger than the limit of 250Mbit/s in at least one place", note.file()->filename());
589 case dcp::VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE:
590 return String::compose("The instantaneous bit rate of the picture asset %1 is close to the limit of 250Mbit/s in at least one place", note.file()->filename());