Store and allow access to the raw XML that is read in from
[libdcp.git] / src / verify.cc
index cb2b12872d75a7630a9b96f56563af1906e8ddd7..c06b7ff9d6d9827586f00266f53633ddd965f848 100644 (file)
@@ -1,5 +1,5 @@
 /*
-    Copyright (C) 2018-2019 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2018-2020 Carl Hetherington <cth@carlh.net>
 
     This file is part of libdcp.
 
 #include "reel.h"
 #include "reel_picture_asset.h"
 #include "reel_sound_asset.h"
+#include "mono_picture_asset.h"
+#include "mono_picture_frame.h"
+#include "stereo_picture_asset.h"
+#include "stereo_picture_frame.h"
 #include "exceptions.h"
 #include "compose.hpp"
 #include "raw_convert.h"
@@ -60,7 +64,6 @@
 #include <boost/noncopyable.hpp>
 #include <boost/foreach.hpp>
 #include <boost/algorithm/string.hpp>
-#include <boost/regex.hpp>
 #include <map>
 #include <list>
 #include <vector>
@@ -71,19 +74,15 @@ using std::vector;
 using std::string;
 using std::cout;
 using std::map;
+using std::max;
 using boost::shared_ptr;
 using boost::optional;
 using boost::function;
+using boost::dynamic_pointer_cast;
 
 using namespace dcp;
 using namespace xercesc;
 
-enum Result {
-       RESULT_GOOD,
-       RESULT_CPL_PKL_DIFFER,
-       RESULT_BAD
-};
-
 static
 string
 xml_ch_to_string (XMLCh const * a)
@@ -155,15 +154,8 @@ private:
        {
                /* XXX: nasty hack */
                if (
-                       e.message() ==
-                       "schema document '/home/carl/src/libdcp/xsd/xml.xsd' has different target namespace "
-                       "from the one specified in instance document 'http://www.w3.org/2001/03/xml.xsd'" ||
-                       e.message() ==
-                       "schema document '/home/carl/src/libdcp/xsd/xmldsig-core-schema.xsd' has different target namespace "
-                       "from the one specified in instance document 'http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd'" ||
-                       e.message() ==
-                       "schema document '/home/carl/src/libdcp/xsd/SMPTE-429-8-2006-PKL.xsd' has different target namespace "
-                       "from the one specified in instance document 'http://www.smpte-ra.org/schemas/429-8/2006/PKL'"
+                       e.message().find("schema document") != string::npos &&
+                       e.message().find("has different target namespace from the one specified in instance document") != string::npos
                        ) {
                        return;
                }
@@ -204,6 +196,10 @@ public:
                add("http://www.w3.org/2001/XMLSchema.dtd", "XMLSchema.dtd");
                add("http://www.w3.org/2001/03/xml.xsd", "xml.xsd");
                add("http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd", "xmldsig-core-schema.xsd");
+               add("http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd", "Main-Stereo-Picture-CPL.xsd");
+               add("http://www.digicine.com/PROTO-ASDCP-CPL-20040511.xsd", "PROTO-ASDCP-CPL-20040511.xsd");
+               add("http://www.digicine.com/PROTO-ASDCP-PKL-20040311.xsd", "PROTO-ASDCP-PKL-20040311.xsd");
+               add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd");
        }
 
        InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id)
@@ -253,7 +249,10 @@ validate_xml (boost::filesystem::path xml_file, boost::filesystem::path xsd_dtd_
                schema["http://www.smpte-ra.org/schemas/429-7/2006/CPL"] = "SMPTE-429-7-2006-CPL.xsd";
                schema["http://www.smpte-ra.org/schemas/429-8/2006/PKL"] = "SMPTE-429-8-2006-PKL.xsd";
                schema["http://www.smpte-ra.org/schemas/429-9/2007/AM"] = "SMPTE-429-9-2007-AM.xsd";
-               schema["http://www.w3.org/2001/03/xml.xsd"] = "xml.xsd";
+               schema["http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd"] = "Main-Stereo-Picture-CPL.xsd";
+               schema["http://www.digicine.com/PROTO-ASDCP-CPL-20040511#"] = "PROTO-ASDCP-CPL-20040511.xsd";
+               schema["http://www.digicine.com/PROTO-ASDCP-PKL-20040311#"] = "PROTO-ASDCP-PKL-20040311.xsd";
+               schema["http://www.digicine.com/PROTO-ASDCP-AM-20040311#"] = "PROTO-ASDCP-AM-20040311.xsd";
 
                string locations;
                for (map<string, string>::const_iterator i = schema.begin(); i != schema.end(); ++i) {
@@ -288,7 +287,7 @@ validate_xml (boost::filesystem::path xml_file, boost::filesystem::path xsd_dtd_
                notes.push_back (
                        VerificationNote(
                                VerificationNote::VERIFY_ERROR,
-                               VerificationNote::Code::XML_VALIDATION_ERROR,
+                               VerificationNote::XML_VALIDATION_ERROR,
                                i.message(),
                                xml_file,
                                i.line()
@@ -297,8 +296,16 @@ validate_xml (boost::filesystem::path xml_file, boost::filesystem::path xsd_dtd_
        }
 }
 
-static Result
-verify_asset (shared_ptr<DCP> dcp, shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
+
+enum VerifyAssetResult {
+       VERIFY_ASSET_RESULT_GOOD,
+       VERIFY_ASSET_RESULT_CPL_PKL_DIFFER,
+       VERIFY_ASSET_RESULT_BAD
+};
+
+
+static VerifyAssetResult
+verify_asset (shared_ptr<const DCP> dcp, shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
 {
        string const actual_hash = reel_mxf->asset_ref()->hash(progress);
 
@@ -320,14 +327,163 @@ verify_asset (shared_ptr<DCP> dcp, shared_ptr<ReelMXF> reel_mxf, function<void (
 
        optional<string> cpl_hash = reel_mxf->hash();
        if (cpl_hash && *cpl_hash != *pkl_hash) {
-               return RESULT_CPL_PKL_DIFFER;
+               return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER;
        }
 
        if (actual_hash != *pkl_hash) {
-               return RESULT_BAD;
+               return VERIFY_ASSET_RESULT_BAD;
+       }
+
+       return VERIFY_ASSET_RESULT_GOOD;
+}
+
+
+enum VerifyPictureAssetResult
+{
+       VERIFY_PICTURE_ASSET_RESULT_GOOD,
+       VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG,
+       VERIFY_PICTURE_ASSET_RESULT_BAD,
+};
+
+
+int
+biggest_frame_size (shared_ptr<const MonoPictureFrame> frame)
+{
+       return frame->j2k_size ();
+}
+
+int
+biggest_frame_size (shared_ptr<const StereoPictureFrame> frame)
+{
+       return max(frame->left_j2k_size(), frame->right_j2k_size());
+}
+
+
+template <class A, class R, class F>
+optional<VerifyPictureAssetResult>
+verify_picture_asset_type (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
+{
+       shared_ptr<A> asset = dynamic_pointer_cast<A>(reel_mxf->asset_ref().asset());
+       if (!asset) {
+               return optional<VerifyPictureAssetResult>();
+       }
+
+       int biggest_frame = 0;
+       shared_ptr<R> reader = asset->start_read ();
+       int64_t const duration = asset->intrinsic_duration ();
+       for (int64_t i = 0; i < duration; ++i) {
+               shared_ptr<const F> frame = reader->get_frame (i);
+               biggest_frame = max(biggest_frame, biggest_frame_size(frame));
+               progress (float(i) / duration);
        }
 
-       return RESULT_GOOD;
+       static const int max_frame =   rint(250 * 1000000 / (8 * asset->edit_rate().as_float()));
+       static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float()));
+       if (biggest_frame > max_frame) {
+               return VERIFY_PICTURE_ASSET_RESULT_BAD;
+       } else if (biggest_frame > risky_frame) {
+               return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG;
+       }
+
+       return VERIFY_PICTURE_ASSET_RESULT_GOOD;
+}
+
+
+static VerifyPictureAssetResult
+verify_picture_asset (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
+{
+       optional<VerifyPictureAssetResult> r = verify_picture_asset_type<MonoPictureAsset, MonoPictureAssetReader, MonoPictureFrame>(reel_mxf, progress);
+       if (!r) {
+               r = verify_picture_asset_type<StereoPictureAsset, StereoPictureAssetReader, StereoPictureFrame>(reel_mxf, progress);
+       }
+
+       DCP_ASSERT (r);
+       return *r;
+}
+
+
+static void
+verify_main_picture_asset (
+       shared_ptr<const DCP> dcp,
+       shared_ptr<const Reel> reel,
+       function<void (string, optional<boost::filesystem::path>)> stage,
+       function<void (float)> progress,
+       list<VerificationNote>& notes
+       )
+{
+       boost::filesystem::path const file = *reel->main_picture()->asset()->file();
+       stage ("Checking picture asset hash", file);
+       VerifyAssetResult const r = verify_asset (dcp, reel->main_picture(), progress);
+       switch (r) {
+               case VERIFY_ASSET_RESULT_BAD:
+                       notes.push_back (
+                               VerificationNote(
+                                       VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, file
+                                       )
+                               );
+                       break;
+               case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
+                       notes.push_back (
+                               VerificationNote(
+                                       VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE, file
+                                       )
+                               );
+                       break;
+               default:
+                       break;
+       }
+       stage ("Checking picture frame sizes", reel->main_picture()->asset()->file());
+       VerifyPictureAssetResult const pr = verify_picture_asset (reel->main_picture(), progress);
+       switch (pr) {
+               case VERIFY_PICTURE_ASSET_RESULT_BAD:
+                       notes.push_back (
+                               VerificationNote(
+                                       VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_FRAME_TOO_LARGE, file
+                                       )
+                               );
+                       break;
+               case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG:
+                       notes.push_back (
+                               VerificationNote(
+                                       VerificationNote::VERIFY_WARNING, VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE, file
+                                       )
+                               );
+                       break;
+               default:
+                       break;
+       }
+}
+
+
+static void
+verify_main_sound_asset (
+       shared_ptr<const DCP> dcp,
+       shared_ptr<const Reel> reel,
+       function<void (string, optional<boost::filesystem::path>)> stage,
+       function<void (float)> progress,
+       list<VerificationNote>& notes
+       )
+{
+       stage ("Checking sound asset hash", reel->main_sound()->asset()->file());
+       VerifyAssetResult const r = verify_asset (dcp, reel->main_sound(), progress);
+       switch (r) {
+               case VERIFY_ASSET_RESULT_BAD:
+                       notes.push_back (
+                               VerificationNote(
+                                       VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *reel->main_sound()->asset()->file()
+                                       )
+                               );
+                       break;
+               case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
+                       notes.push_back (
+                               VerificationNote(
+                                       VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE, *reel->main_sound()->asset()->file()
+                                       )
+                               );
+                       break;
+               default:
+                       break;
+       }
 }
 
 
@@ -352,10 +508,10 @@ dcp::verify (
                stage ("Checking DCP", dcp->directory());
                try {
                        dcp->read (&notes);
-               } catch (DCPReadError& e) {
-                       notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::Code::GENERAL_READ, string(e.what())));
+               } catch (ReadError& e) {
+                       notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
                } catch (XMLError& e) {
-                       notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::Code::GENERAL_READ, string(e.what())));
+                       notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
                }
 
                BOOST_FOREACH (shared_ptr<CPL> cpl, dcp->cpls()) {
@@ -372,6 +528,16 @@ dcp::verify (
 
                        BOOST_FOREACH (shared_ptr<Reel> reel, cpl->reels()) {
                                stage ("Checking reel", optional<boost::filesystem::path>());
+
+                               BOOST_FOREACH (shared_ptr<ReelAsset> i, reel->assets()) {
+                                       if (i->duration() && (i->duration().get() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
+                                               notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::DURATION_TOO_SMALL, i->id()));
+                                       }
+                                       if ((i->intrinsic_duration() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
+                                               notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INTRINSIC_DURATION_TOO_SMALL, i->id()));
+                                       }
+                               }
+
                                if (reel->main_picture()) {
                                        /* Check reel stuff */
                                        Fraction const frame_rate = reel->main_picture()->frame_rate();
@@ -387,49 +553,11 @@ dcp::verify (
                                        }
                                        /* Check asset */
                                        if (reel->main_picture()->asset_ref().resolved()) {
-                                               stage ("Checking picture asset hash", reel->main_picture()->asset()->file());
-                                               Result const r = verify_asset (dcp, reel->main_picture(), progress);
-                                               switch (r) {
-                                               case RESULT_BAD:
-                                                       notes.push_back (
-                                                               VerificationNote(
-                                                                       VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, *reel->main_picture()->asset()->file()
-                                                                       )
-                                                               );
-                                                       break;
-                                               case RESULT_CPL_PKL_DIFFER:
-                                                       notes.push_back (
-                                                               VerificationNote(
-                                                                       VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE, *reel->main_picture()->asset()->file()
-                                                                       )
-                                                               );
-                                                       break;
-                                               default:
-                                                       break;
-                                               }
+                                               verify_main_picture_asset (dcp, reel, stage, progress, notes);
                                        }
                                }
                                if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
-                                       stage ("Checking sound asset hash", reel->main_sound()->asset()->file());
-                                       Result const r = verify_asset (dcp, reel->main_sound(), progress);
-                                       switch (r) {
-                                       case RESULT_BAD:
-                                               notes.push_back (
-                                                       VerificationNote(
-                                                               VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *reel->main_sound()->asset()->file()
-                                                               )
-                                                       );
-                                               break;
-                                       case RESULT_CPL_PKL_DIFFER:
-                                               notes.push_back (
-                                                       VerificationNote(
-                                                               VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE, *reel->main_sound()->asset()->file()
-                                                               )
-                                                       );
-                                               break;
-                                       default:
-                                               break;
-                                       }
+                                       verify_main_sound_asset (dcp, reel, stage, progress, notes);
                                }
                        }
                }
@@ -439,9 +567,12 @@ dcp::verify (
                        validate_xml (pkl->file().get(), xsd_dtd_directory, notes);
                }
 
-               stage ("Checking ASSETMAP", dcp->asset_map_path().get());
-               validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
-
+               if (dcp->asset_map_path()) {
+                       stage ("Checking ASSETMAP", dcp->asset_map_path().get());
+                       validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
+               } else {
+                       notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::MISSING_ASSETMAP));
+               }
        }
 
        return notes;
@@ -473,6 +604,16 @@ dcp::note_to_string (dcp::VerificationNote note)
                return "The DCP contains both SMPTE and Interop parts.";
        case dcp::VerificationNote::XML_VALIDATION_ERROR:
                return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get());
+       case dcp::VerificationNote::MISSING_ASSETMAP:
+               return "No ASSETMAP or ASSETMAP.xml was found";
+       case dcp::VerificationNote::INTRINSIC_DURATION_TOO_SMALL:
+               return String::compose("The intrinsic duration of an asset is less than 1 second long: %1", note.note().get());
+       case dcp::VerificationNote::DURATION_TOO_SMALL:
+               return String::compose("The duration of an asset is less than 1 second long: %1", note.note().get());
+       case dcp::VerificationNote::PICTURE_FRAME_TOO_LARGE:
+               return String::compose("The instantaneous bit rate of the picture asset %1 is larger than the limit of 250Mbit/s in at least one place", note.file()->filename());
+       case dcp::VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE:
+               return String::compose("The instantaneous bit rate of the picture asset %1 is close to the limit of 250Mbit/s in at least one place", note.file()->filename());
        }
 
        return "";