src/verify.cc

   1 /*
   2     Copyright (C) 2018-2019 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of libdcp.
   5
   6     libdcp is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     libdcp is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with libdcp.  If not, see <http://www.gnu.org/licenses/>.
  18
  19     In addition, as a special exception, the copyright holders give
  20     permission to link the code of portions of this program with the
  21     OpenSSL library under certain conditions as described in each
  22     individual source file, and distribute linked combinations
  23     including the two.
  24
  25     You must obey the GNU General Public License in all respects
  26     for all of the code used other than OpenSSL.  If you modify
  27     file(s) with this exception, you may extend this exception to your
  28     version of the file(s), but you are not obligated to do so.  If you
  29     do not wish to do so, delete this exception statement from your
  30     version.  If you delete this exception statement from all source
  31     files in the program, then also delete it here.
  32 */
  33
  34 #include "verify.h"
  35 #include "dcp.h"
  36 #include "cpl.h"
  37 #include "reel.h"
  38 #include "reel_picture_asset.h"
  39 #include "reel_sound_asset.h"
  40 #include "exceptions.h"
  41 #include "compose.hpp"
  42 #include "raw_convert.h"
  43 #include <xercesc/util/PlatformUtils.hpp>
  44 #include <xercesc/parsers/XercesDOMParser.hpp>
  45 #include <xercesc/parsers/AbstractDOMParser.hpp>
  46 #include <xercesc/sax/HandlerBase.hpp>
  47 #include <xercesc/dom/DOMImplementation.hpp>
  48 #include <xercesc/dom/DOMImplementationLS.hpp>
  49 #include <xercesc/dom/DOMImplementationRegistry.hpp>
  50 #include <xercesc/dom/DOMLSParser.hpp>
  51 #include <xercesc/dom/DOMException.hpp>
  52 #include <xercesc/dom/DOMDocument.hpp>
  53 #include <xercesc/dom/DOMNodeList.hpp>
  54 #include <xercesc/dom/DOMError.hpp>
  55 #include <xercesc/dom/DOMLocator.hpp>
  56 #include <xercesc/dom/DOMNamedNodeMap.hpp>
  57 #include <xercesc/dom/DOMAttr.hpp>
  58 #include <xercesc/dom/DOMErrorHandler.hpp>
  59 #include <xercesc/framework/LocalFileInputSource.hpp>
  60 #include <boost/noncopyable.hpp>
  61 #include <boost/foreach.hpp>
  62 #include <boost/algorithm/string.hpp>
  63 #include <map>
  64 #include <list>
  65 #include <vector>
  66 #include <iostream>
  67
  68 using std::list;
  69 using std::vector;
  70 using std::string;
  71 using std::cout;
  72 using std::map;
  73 using boost::shared_ptr;
  74 using boost::optional;
  75 using boost::function;
  76
  77 using namespace dcp;
  78 using namespace xercesc;
  79
  80 enum Result {
  81         RESULT_GOOD,
  82         RESULT_CPL_PKL_DIFFER,
  83         RESULT_BAD
  84 };
  85
  86 static
  87 string
  88 xml_ch_to_string (XMLCh const * a)
  89 {
  90         char* x = XMLString::transcode(a);
  91         string const o(x);
  92         XMLString::release(&x);
  93         return o;
  94 }
  95
  96 class XMLValidationError
  97 {
  98 public:
  99         XMLValidationError (SAXParseException const & e)
 100                 : _message (xml_ch_to_string(e.getMessage()))
 101                 , _line (e.getLineNumber())
 102                 , _column (e.getColumnNumber())
 103         {
 104
 105         }
 106
 107         string message () const {
 108                 return _message;
 109         }
 110
 111         uint64_t line () const {
 112                 return _line;
 113         }
 114
 115         uint64_t column () const {
 116                 return _column;
 117         }
 118
 119 private:
 120         string _message;
 121         uint64_t _line;
 122         uint64_t _column;
 123 };
 124
 125
 126 class DCPErrorHandler : public ErrorHandler
 127 {
 128 public:
 129         void warning(const SAXParseException& e)
 130         {
 131                 maybe_add (XMLValidationError(e));
 132         }
 133
 134         void error(const SAXParseException& e)
 135         {
 136                 maybe_add (XMLValidationError(e));
 137         }
 138
 139         void fatalError(const SAXParseException& e)
 140         {
 141                 maybe_add (XMLValidationError(e));
 142         }
 143
 144         void resetErrors() {
 145                 _errors.clear ();
 146         }
 147
 148         list<XMLValidationError> errors () const {
 149                 return _errors;
 150         }
 151
 152 private:
 153         void maybe_add (XMLValidationError e)
 154         {
 155                 /* XXX: nasty hack */
 156                 if (
 157                         e.message().find("schema document") != string::npos &&
 158                         e.message().find("has different target namespace from the one specified in instance document") != string::npos
 159                         ) {
 160                         return;
 161                 }
 162
 163                 _errors.push_back (e);
 164         }
 165
 166         list<XMLValidationError> _errors;
 167 };
 168
 169 class StringToXMLCh : public boost::noncopyable
 170 {
 171 public:
 172         StringToXMLCh (string a)
 173         {
 174                 _buffer = XMLString::transcode(a.c_str());
 175         }
 176
 177         ~StringToXMLCh ()
 178         {
 179                 XMLString::release (&_buffer);
 180         }
 181
 182         XMLCh const * get () const {
 183                 return _buffer;
 184         }
 185
 186 private:
 187         XMLCh* _buffer;
 188 };
 189
 190 class LocalFileResolver : public EntityResolver
 191 {
 192 public:
 193         LocalFileResolver (boost::filesystem::path xsd_dtd_directory)
 194                 : _xsd_dtd_directory (xsd_dtd_directory)
 195         {
 196                 add("http://www.w3.org/2001/XMLSchema.dtd", "XMLSchema.dtd");
 197                 add("http://www.w3.org/2001/03/xml.xsd", "xml.xsd");
 198                 add("http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd", "xmldsig-core-schema.xsd");
 199         }
 200
 201         InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id)
 202         {
 203                 string system_id_str = xml_ch_to_string (system_id);
 204                 if (_files.find(system_id_str) == _files.end()) {
 205                         return 0;
 206                 }
 207
 208                 boost::filesystem::path p = _xsd_dtd_directory / _files[system_id_str];
 209                 StringToXMLCh ch (p.string());
 210                 return new LocalFileInputSource(ch.get());
 211         }
 212
 213 private:
 214         void add (string uri, string file)
 215         {
 216                 _files[uri] = file;
 217         }
 218
 219         std::map<string, string> _files;
 220         boost::filesystem::path _xsd_dtd_directory;
 221 };
 222
 223 static
 224 void
 225 validate_xml (boost::filesystem::path xml_file, boost::filesystem::path xsd_dtd_directory, list<VerificationNote>& notes)
 226 {
 227         try {
 228                 XMLPlatformUtils::Initialize ();
 229         } catch (XMLException& e) {
 230                 throw MiscError ("Failed to initialise xerces library");
 231         }
 232
 233         DCPErrorHandler error_handler;
 234
 235         /* All the xerces objects in this scope must be destroyed before XMLPlatformUtils::Terminate() is called */
 236         {
 237                 XercesDOMParser parser;
 238                 parser.setValidationScheme(XercesDOMParser::Val_Always);
 239                 parser.setDoNamespaces(true);
 240                 parser.setDoSchema(true);
 241
 242                 map<string, string> schema;
 243                 schema["http://www.w3.org/2000/09/xmldsig#"] = "xmldsig-core-schema.xsd";
 244                 schema["http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd"] = "xmldsig-core-schema.xsd";
 245                 schema["http://www.smpte-ra.org/schemas/429-7/2006/CPL"] = "SMPTE-429-7-2006-CPL.xsd";
 246                 schema["http://www.smpte-ra.org/schemas/429-8/2006/PKL"] = "SMPTE-429-8-2006-PKL.xsd";
 247                 schema["http://www.smpte-ra.org/schemas/429-9/2007/AM"] = "SMPTE-429-9-2007-AM.xsd";
 248                 schema["http://www.w3.org/2001/03/xml.xsd"] = "xml.xsd";
 249
 250                 string locations;
 251                 for (map<string, string>::const_iterator i = schema.begin(); i != schema.end(); ++i) {
 252                         locations += i->first;
 253                         locations += " ";
 254                         boost::filesystem::path p = xsd_dtd_directory / i->second;
 255                         locations += p.string() + " ";
 256                 }
 257
 258                 parser.setExternalSchemaLocation(locations.c_str());
 259                 parser.setValidationSchemaFullChecking(true);
 260                 parser.setErrorHandler(&error_handler);
 261
 262                 LocalFileResolver resolver (xsd_dtd_directory);
 263                 parser.setEntityResolver(&resolver);
 264
 265                 try {
 266                         parser.resetDocumentPool();
 267                         parser.parse(xml_file.string().c_str());
 268                 } catch (XMLException& e) {
 269                         throw MiscError(xml_ch_to_string(e.getMessage()));
 270                 } catch (DOMException& e) {
 271                         throw MiscError(xml_ch_to_string(e.getMessage()));
 272                 } catch (...) {
 273                         throw MiscError("Unknown exception from xerces");
 274                 }
 275         }
 276
 277         XMLPlatformUtils::Terminate ();
 278
 279         BOOST_FOREACH (XMLValidationError i, error_handler.errors()) {
 280                 notes.push_back (
 281                         VerificationNote(
 282                                 VerificationNote::VERIFY_ERROR,
 283                                 VerificationNote::XML_VALIDATION_ERROR,
 284                                 i.message(),
 285                                 xml_file,
 286                                 i.line()
 287                                 )
 288                         );
 289         }
 290 }
 291
 292 static Result
 293 verify_asset (shared_ptr<DCP> dcp, shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
 294 {
 295         string const actual_hash = reel_mxf->asset_ref()->hash(progress);
 296
 297         list<shared_ptr<PKL> > pkls = dcp->pkls();
 298         /* We've read this DCP in so it must have at least one PKL */
 299         DCP_ASSERT (!pkls.empty());
 300
 301         shared_ptr<Asset> asset = reel_mxf->asset_ref().asset();
 302
 303         optional<string> pkl_hash;
 304         BOOST_FOREACH (shared_ptr<PKL> i, pkls) {
 305                 pkl_hash = i->hash (reel_mxf->asset_ref()->id());
 306                 if (pkl_hash) {
 307                         break;
 308                 }
 309         }
 310
 311         DCP_ASSERT (pkl_hash);
 312
 313         optional<string> cpl_hash = reel_mxf->hash();
 314         if (cpl_hash && *cpl_hash != *pkl_hash) {
 315                 return RESULT_CPL_PKL_DIFFER;
 316         }
 317
 318         if (actual_hash != *pkl_hash) {
 319                 return RESULT_BAD;
 320         }
 321
 322         return RESULT_GOOD;
 323 }
 324
 325
 326 list<VerificationNote>
 327 dcp::verify (
 328         vector<boost::filesystem::path> directories,
 329         function<void (string, optional<boost::filesystem::path>)> stage,
 330         function<void (float)> progress,
 331         boost::filesystem::path xsd_dtd_directory
 332         )
 333 {
 334         xsd_dtd_directory = boost::filesystem::canonical (xsd_dtd_directory);
 335
 336         list<VerificationNote> notes;
 337
 338         list<shared_ptr<DCP> > dcps;
 339         BOOST_FOREACH (boost::filesystem::path i, directories) {
 340                 dcps.push_back (shared_ptr<DCP> (new DCP (i)));
 341         }
 342
 343         BOOST_FOREACH (shared_ptr<DCP> dcp, dcps) {
 344                 stage ("Checking DCP", dcp->directory());
 345                 try {
 346                         dcp->read (&notes);
 347                 } catch (DCPReadError& e) {
 348                         notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
 349                 } catch (XMLError& e) {
 350                         notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
 351                 }
 352
 353                 BOOST_FOREACH (shared_ptr<CPL> cpl, dcp->cpls()) {
 354                         stage ("Checking CPL", cpl->file());
 355                         validate_xml (cpl->file().get(), xsd_dtd_directory, notes);
 356
 357                         /* Check that the CPL's hash corresponds to the PKL */
 358                         BOOST_FOREACH (shared_ptr<PKL> i, dcp->pkls()) {
 359                                 optional<string> h = i->hash(cpl->id());
 360                                 if (h && make_digest(Data(*cpl->file())) != *h) {
 361                                         notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::CPL_HASH_INCORRECT));
 362                                 }
 363                         }
 364
 365                         BOOST_FOREACH (shared_ptr<Reel> reel, cpl->reels()) {
 366                                 stage ("Checking reel", optional<boost::filesystem::path>());
 367                                 if (reel->main_picture()) {
 368                                         /* Check reel stuff */
 369                                         Fraction const frame_rate = reel->main_picture()->frame_rate();
 370                                         if (frame_rate.denominator != 1 ||
 371                                             (frame_rate.numerator != 24 &&
 372                                              frame_rate.numerator != 25 &&
 373                                              frame_rate.numerator != 30 &&
 374                                              frame_rate.numerator != 48 &&
 375                                              frame_rate.numerator != 50 &&
 376                                              frame_rate.numerator != 60 &&
 377                                              frame_rate.numerator != 96)) {
 378                                                 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INVALID_PICTURE_FRAME_RATE));
 379                                         }
 380                                         /* Check asset */
 381                                         if (reel->main_picture()->asset_ref().resolved()) {
 382                                                 stage ("Checking picture asset hash", reel->main_picture()->asset()->file());
 383                                                 Result const r = verify_asset (dcp, reel->main_picture(), progress);
 384                                                 switch (r) {
 385                                                 case RESULT_BAD:
 386                                                         notes.push_back (
 387                                                                 VerificationNote(
 388                                                                         VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, *reel->main_picture()->asset()->file()
 389                                                                         )
 390                                                                 );
 391                                                         break;
 392                                                 case RESULT_CPL_PKL_DIFFER:
 393                                                         notes.push_back (
 394                                                                 VerificationNote(
 395                                                                         VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE, *reel->main_picture()->asset()->file()
 396                                                                         )
 397                                                                 );
 398                                                         break;
 399                                                 default:
 400                                                         break;
 401                                                 }
 402                                         }
 403                                 }
 404                                 if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
 405                                         stage ("Checking sound asset hash", reel->main_sound()->asset()->file());
 406                                         Result const r = verify_asset (dcp, reel->main_sound(), progress);
 407                                         switch (r) {
 408                                         case RESULT_BAD:
 409                                                 notes.push_back (
 410                                                         VerificationNote(
 411                                                                 VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *reel->main_sound()->asset()->file()
 412                                                                 )
 413                                                         );
 414                                                 break;
 415                                         case RESULT_CPL_PKL_DIFFER:
 416                                                 notes.push_back (
 417                                                         VerificationNote(
 418                                                                 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE, *reel->main_sound()->asset()->file()
 419                                                                 )
 420                                                         );
 421                                                 break;
 422                                         default:
 423                                                 break;
 424                                         }
 425                                 }
 426                         }
 427                 }
 428
 429                 BOOST_FOREACH (shared_ptr<PKL> pkl, dcp->pkls()) {
 430                         stage ("Checking PKL", pkl->file());
 431                         validate_xml (pkl->file().get(), xsd_dtd_directory, notes);
 432                 }
 433
 434                 stage ("Checking ASSETMAP", dcp->asset_map_path().get());
 435                 validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
 436
 437         }
 438
 439         return notes;
 440 }
 441
 442 string
 443 dcp::note_to_string (dcp::VerificationNote note)
 444 {
 445         switch (note.code()) {
 446         case dcp::VerificationNote::GENERAL_READ:
 447                 return *note.note();
 448         case dcp::VerificationNote::CPL_HASH_INCORRECT:
 449                 return "The hash of the CPL in the PKL does not agree with the CPL file";
 450         case dcp::VerificationNote::INVALID_PICTURE_FRAME_RATE:
 451                 return "The picture in a reel has an invalid frame rate";
 452         case dcp::VerificationNote::PICTURE_HASH_INCORRECT:
 453                 return dcp::String::compose("The hash of the picture asset %1 does not agree with the PKL file", note.file()->filename());
 454         case dcp::VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE:
 455                 return dcp::String::compose("The PKL and CPL hashes disagree for the picture asset %1", note.file()->filename());
 456         case dcp::VerificationNote::SOUND_HASH_INCORRECT:
 457                 return dcp::String::compose("The hash of the sound asset %1 does not agree with the PKL file", note.file()->filename());
 458         case dcp::VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE:
 459                 return dcp::String::compose("The PKL and CPL hashes disagree for the sound asset %1", note.file()->filename());
 460         case dcp::VerificationNote::EMPTY_ASSET_PATH:
 461                 return "The asset map contains an empty asset path.";
 462         case dcp::VerificationNote::MISSING_ASSET:
 463                 return String::compose("The file for an asset in the asset map cannot be found; missing file is %1.", note.file()->filename());
 464         case dcp::VerificationNote::MISMATCHED_STANDARD:
 465                 return "The DCP contains both SMPTE and Interop parts.";
 466         case dcp::VerificationNote::XML_VALIDATION_ERROR:
 467                 return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get());
 468         }
 469
 470         return "";
 471 }