2 Copyright (C) 2018-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of libdcp.
6 libdcp is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 libdcp is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with libdcp. If not, see <http://www.gnu.org/licenses/>.
19 In addition, as a special exception, the copyright holders give
20 permission to link the code of portions of this program with the
21 OpenSSL library under certain conditions as described in each
22 individual source file, and distribute linked combinations
25 You must obey the GNU General Public License in all respects
26 for all of the code used other than OpenSSL. If you modify
27 file(s) with this exception, you may extend this exception to your
28 version of the file(s), but you are not obligated to do so. If you
29 do not wish to do so, delete this exception statement from your
30 version. If you delete this exception statement from all source
31 files in the program, then also delete it here.
38 #include "reel_closed_caption_asset.h"
39 #include "reel_picture_asset.h"
40 #include "reel_sound_asset.h"
41 #include "reel_subtitle_asset.h"
42 #include "interop_subtitle_asset.h"
43 #include "mono_picture_asset.h"
44 #include "mono_picture_frame.h"
45 #include "stereo_picture_asset.h"
46 #include "stereo_picture_frame.h"
47 #include "exceptions.h"
48 #include "compose.hpp"
49 #include "raw_convert.h"
50 #include "smpte_subtitle_asset.h"
51 #include <xercesc/util/PlatformUtils.hpp>
52 #include <xercesc/parsers/XercesDOMParser.hpp>
53 #include <xercesc/parsers/AbstractDOMParser.hpp>
54 #include <xercesc/sax/HandlerBase.hpp>
55 #include <xercesc/dom/DOMImplementation.hpp>
56 #include <xercesc/dom/DOMImplementationLS.hpp>
57 #include <xercesc/dom/DOMImplementationRegistry.hpp>
58 #include <xercesc/dom/DOMLSParser.hpp>
59 #include <xercesc/dom/DOMException.hpp>
60 #include <xercesc/dom/DOMDocument.hpp>
61 #include <xercesc/dom/DOMNodeList.hpp>
62 #include <xercesc/dom/DOMError.hpp>
63 #include <xercesc/dom/DOMLocator.hpp>
64 #include <xercesc/dom/DOMNamedNodeMap.hpp>
65 #include <xercesc/dom/DOMAttr.hpp>
66 #include <xercesc/dom/DOMErrorHandler.hpp>
67 #include <xercesc/framework/LocalFileInputSource.hpp>
68 #include <xercesc/framework/MemBufInputSource.hpp>
69 #include <boost/noncopyable.hpp>
70 #include <boost/algorithm/string.hpp>
81 using std::shared_ptr;
82 using std::make_shared;
83 using boost::optional;
84 using boost::function;
85 using std::dynamic_pointer_cast;
88 using namespace xercesc;
92 xml_ch_to_string (XMLCh const * a)
94 char* x = XMLString::transcode(a);
96 XMLString::release(&x);
100 class XMLValidationError
103 XMLValidationError (SAXParseException const & e)
104 : _message (xml_ch_to_string(e.getMessage()))
105 , _line (e.getLineNumber())
106 , _column (e.getColumnNumber())
107 , _public_id (e.getPublicId() ? xml_ch_to_string(e.getPublicId()) : "")
108 , _system_id (e.getSystemId() ? xml_ch_to_string(e.getSystemId()) : "")
113 string message () const {
117 uint64_t line () const {
121 uint64_t column () const {
125 string public_id () const {
129 string system_id () const {
142 class DCPErrorHandler : public ErrorHandler
145 void warning(const SAXParseException& e)
147 maybe_add (XMLValidationError(e));
150 void error(const SAXParseException& e)
152 maybe_add (XMLValidationError(e));
155 void fatalError(const SAXParseException& e)
157 maybe_add (XMLValidationError(e));
164 list<XMLValidationError> errors () const {
169 void maybe_add (XMLValidationError e)
171 /* XXX: nasty hack */
173 e.message().find("schema document") != string::npos &&
174 e.message().find("has different target namespace from the one specified in instance document") != string::npos
179 _errors.push_back (e);
182 list<XMLValidationError> _errors;
185 class StringToXMLCh : public boost::noncopyable
188 StringToXMLCh (string a)
190 _buffer = XMLString::transcode(a.c_str());
195 XMLString::release (&_buffer);
198 XMLCh const * get () const {
206 class LocalFileResolver : public EntityResolver
209 LocalFileResolver (boost::filesystem::path xsd_dtd_directory)
210 : _xsd_dtd_directory (xsd_dtd_directory)
212 /* XXX: I'm not clear on what things need to be in this list; some XSDs are apparently, magically
213 * found without being here.
215 add("http://www.w3.org/2001/XMLSchema.dtd", "XMLSchema.dtd");
216 add("http://www.w3.org/2001/03/xml.xsd", "xml.xsd");
217 add("http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd", "xmldsig-core-schema.xsd");
218 add("http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd", "Main-Stereo-Picture-CPL.xsd");
219 add("http://www.digicine.com/PROTO-ASDCP-CPL-20040511.xsd", "PROTO-ASDCP-CPL-20040511.xsd");
220 add("http://www.digicine.com/PROTO-ASDCP-PKL-20040311.xsd", "PROTO-ASDCP-PKL-20040311.xsd");
221 add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd");
222 add("http://www.digicine.com/PROTO-ASDCP-CC-CPL-20070926#", "PROTO-ASDCP-CC-CPL-20070926.xsd");
223 add("interop-subs", "DCSubtitle.v1.mattsson.xsd");
224 add("http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd", "SMPTE-428-7-2010-DCST.xsd");
225 add("http://www.smpte-ra.org/schemas/429-16/2014/CPL-Metadata", "SMPTE-429-16.xsd");
226 add("http://www.dolby.com/schemas/2012/AD", "Dolby-2012-AD.xsd");
227 add("http://www.smpte-ra.org/schemas/429-10/2008/Main-Stereo-Picture-CPL", "SMPTE-429-10-2008.xsd");
230 InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id)
235 auto system_id_str = xml_ch_to_string (system_id);
236 auto p = _xsd_dtd_directory;
237 if (_files.find(system_id_str) == _files.end()) {
240 p /= _files[system_id_str];
242 StringToXMLCh ch (p.string());
243 return new LocalFileInputSource(ch.get());
247 void add (string uri, string file)
252 std::map<string, string> _files;
253 boost::filesystem::path _xsd_dtd_directory;
258 parse (XercesDOMParser& parser, boost::filesystem::path xml)
260 parser.parse(xml.string().c_str());
265 parse (XercesDOMParser& parser, string xml)
267 xercesc::MemBufInputSource buf(reinterpret_cast<unsigned char const*>(xml.c_str()), xml.size(), "");
274 validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, vector<VerificationNote>& notes)
277 XMLPlatformUtils::Initialize ();
278 } catch (XMLException& e) {
279 throw MiscError ("Failed to initialise xerces library");
282 DCPErrorHandler error_handler;
284 /* All the xerces objects in this scope must be destroyed before XMLPlatformUtils::Terminate() is called */
286 XercesDOMParser parser;
287 parser.setValidationScheme(XercesDOMParser::Val_Always);
288 parser.setDoNamespaces(true);
289 parser.setDoSchema(true);
291 vector<string> schema;
292 schema.push_back("xml.xsd");
293 schema.push_back("xmldsig-core-schema.xsd");
294 schema.push_back("SMPTE-429-7-2006-CPL.xsd");
295 schema.push_back("SMPTE-429-8-2006-PKL.xsd");
296 schema.push_back("SMPTE-429-9-2007-AM.xsd");
297 schema.push_back("Main-Stereo-Picture-CPL.xsd");
298 schema.push_back("PROTO-ASDCP-CPL-20040511.xsd");
299 schema.push_back("PROTO-ASDCP-PKL-20040311.xsd");
300 schema.push_back("PROTO-ASDCP-AM-20040311.xsd");
301 schema.push_back("DCSubtitle.v1.mattsson.xsd");
302 schema.push_back("DCDMSubtitle-2010.xsd");
303 schema.push_back("PROTO-ASDCP-CC-CPL-20070926.xsd");
304 schema.push_back("SMPTE-429-16.xsd");
305 schema.push_back("Dolby-2012-AD.xsd");
306 schema.push_back("SMPTE-429-10-2008.xsd");
307 schema.push_back("xlink.xsd");
308 schema.push_back("SMPTE-335-2012.xsd");
309 schema.push_back("SMPTE-395-2014-13-1-aaf.xsd");
310 schema.push_back("isdcf-mca.xsd");
311 schema.push_back("SMPTE-429-12-2008.xsd");
313 /* XXX: I'm not especially clear what this is for, but it seems to be necessary.
314 * Schemas that are not mentioned in this list are not read, and the things
315 * they describe are not checked.
318 for (auto i: schema) {
319 locations += String::compose("%1 %1 ", i, i);
322 parser.setExternalSchemaLocation(locations.c_str());
323 parser.setValidationSchemaFullChecking(true);
324 parser.setErrorHandler(&error_handler);
326 LocalFileResolver resolver (xsd_dtd_directory);
327 parser.setEntityResolver(&resolver);
330 parser.resetDocumentPool();
332 } catch (XMLException& e) {
333 throw MiscError(xml_ch_to_string(e.getMessage()));
334 } catch (DOMException& e) {
335 throw MiscError(xml_ch_to_string(e.getMessage()));
337 throw MiscError("Unknown exception from xerces");
341 XMLPlatformUtils::Terminate ();
343 for (auto i: error_handler.errors()) {
346 VerificationNote::VERIFY_ERROR,
347 VerificationNote::XML_VALIDATION_ERROR,
349 boost::trim_copy(i.public_id() + " " + i.system_id()),
357 enum VerifyAssetResult {
358 VERIFY_ASSET_RESULT_GOOD,
359 VERIFY_ASSET_RESULT_CPL_PKL_DIFFER,
360 VERIFY_ASSET_RESULT_BAD
364 static VerifyAssetResult
365 verify_asset (shared_ptr<const DCP> dcp, shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
367 auto const actual_hash = reel_mxf->asset_ref()->hash(progress);
369 auto pkls = dcp->pkls();
370 /* We've read this DCP in so it must have at least one PKL */
371 DCP_ASSERT (!pkls.empty());
373 auto asset = reel_mxf->asset_ref().asset();
375 optional<string> pkl_hash;
377 pkl_hash = i->hash (reel_mxf->asset_ref()->id());
383 DCP_ASSERT (pkl_hash);
385 auto cpl_hash = reel_mxf->hash();
386 if (cpl_hash && *cpl_hash != *pkl_hash) {
387 return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER;
390 if (actual_hash != *pkl_hash) {
391 return VERIFY_ASSET_RESULT_BAD;
394 return VERIFY_ASSET_RESULT_GOOD;
399 verify_language_tag (string tag, vector<VerificationNote>& notes)
402 dcp::LanguageTag test (tag);
403 } catch (dcp::LanguageTagError &) {
404 notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::BAD_LANGUAGE, tag));
409 enum VerifyPictureAssetResult
411 VERIFY_PICTURE_ASSET_RESULT_GOOD,
412 VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE,
413 VERIFY_PICTURE_ASSET_RESULT_BAD,
418 biggest_frame_size (shared_ptr<const MonoPictureFrame> frame)
420 return frame->size ();
424 biggest_frame_size (shared_ptr<const StereoPictureFrame> frame)
426 return max(frame->left()->size(), frame->right()->size());
430 template <class A, class R, class F>
431 optional<VerifyPictureAssetResult>
432 verify_picture_asset_type (shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
434 auto asset = dynamic_pointer_cast<A>(reel_mxf->asset_ref().asset());
436 return optional<VerifyPictureAssetResult>();
439 int biggest_frame = 0;
440 auto reader = asset->start_read ();
441 auto const duration = asset->intrinsic_duration ();
442 for (int64_t i = 0; i < duration; ++i) {
443 shared_ptr<const F> frame = reader->get_frame (i);
444 biggest_frame = max(biggest_frame, biggest_frame_size(frame));
445 progress (float(i) / duration);
448 static const int max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float()));
449 static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float()));
450 if (biggest_frame > max_frame) {
451 return VERIFY_PICTURE_ASSET_RESULT_BAD;
452 } else if (biggest_frame > risky_frame) {
453 return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE;
456 return VERIFY_PICTURE_ASSET_RESULT_GOOD;
460 static VerifyPictureAssetResult
461 verify_picture_asset (shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
463 auto r = verify_picture_asset_type<MonoPictureAsset, MonoPictureAssetReader, MonoPictureFrame>(reel_mxf, progress);
465 r = verify_picture_asset_type<StereoPictureAsset, StereoPictureAssetReader, StereoPictureFrame>(reel_mxf, progress);
474 verify_main_picture_asset (
475 shared_ptr<const DCP> dcp,
476 shared_ptr<const ReelPictureAsset> reel_asset,
477 function<void (string, optional<boost::filesystem::path>)> stage,
478 function<void (float)> progress,
479 vector<VerificationNote>& notes
482 auto asset = reel_asset->asset();
483 auto const file = *asset->file();
484 stage ("Checking picture asset hash", file);
485 auto const r = verify_asset (dcp, reel_asset, progress);
487 case VERIFY_ASSET_RESULT_BAD:
490 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, file
494 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
497 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DIFFER, file
504 stage ("Checking picture frame sizes", asset->file());
505 auto const pr = verify_picture_asset (reel_asset, progress);
507 case VERIFY_PICTURE_ASSET_RESULT_BAD:
510 VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_FRAME_TOO_LARGE_IN_BYTES, file
514 case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE:
517 VerificationNote::VERIFY_WARNING, VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE_IN_BYTES, file
525 /* Only flat/scope allowed by Bv2.1 */
527 asset->size() != dcp::Size(2048, 858) &&
528 asset->size() != dcp::Size(1998, 1080) &&
529 asset->size() != dcp::Size(4096, 1716) &&
530 asset->size() != dcp::Size(3996, 2160)) {
533 VerificationNote::VERIFY_BV21_ERROR,
534 VerificationNote::PICTURE_ASSET_INVALID_SIZE_IN_PIXELS,
535 String::compose("%1x%2", asset->size().width, asset->size().height),
541 /* Only 24, 25, 48fps allowed for 2K */
543 (asset->size() == dcp::Size(2048, 858) || asset->size() == dcp::Size(1998, 1080)) &&
544 (asset->edit_rate() != dcp::Fraction(24, 1) && asset->edit_rate() != dcp::Fraction(25, 1) && asset->edit_rate() != dcp::Fraction(48, 1))
548 VerificationNote::VERIFY_BV21_ERROR,
549 VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_2K,
550 String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator),
556 if (asset->size() == dcp::Size(4096, 1716) || asset->size() == dcp::Size(3996, 2160)) {
557 /* Only 24fps allowed for 4K */
558 if (asset->edit_rate() != dcp::Fraction(24, 1)) {
561 VerificationNote::VERIFY_BV21_ERROR,
562 VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_4K,
563 String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator),
569 /* Only 2D allowed for 4K */
570 if (dynamic_pointer_cast<const StereoPictureAsset>(asset)) {
573 VerificationNote::VERIFY_BV21_ERROR,
574 VerificationNote::PICTURE_ASSET_4K_3D,
586 verify_main_sound_asset (
587 shared_ptr<const DCP> dcp,
588 shared_ptr<const ReelSoundAsset> reel_asset,
589 function<void (string, optional<boost::filesystem::path>)> stage,
590 function<void (float)> progress,
591 vector<VerificationNote>& notes
594 auto asset = reel_asset->asset();
595 stage ("Checking sound asset hash", asset->file());
596 auto const r = verify_asset (dcp, reel_asset, progress);
598 case VERIFY_ASSET_RESULT_BAD:
601 VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *asset->file()
605 case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
608 VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DIFFER, *asset->file()
616 stage ("Checking sound asset metadata", asset->file());
618 verify_language_tag (asset->language(), notes);
619 if (asset->sampling_rate() != 48000) {
622 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::INVALID_SOUND_FRAME_RATE, *asset->file()
630 verify_main_subtitle_reel (shared_ptr<const ReelSubtitleAsset> reel_asset, vector<VerificationNote>& notes)
632 /* XXX: is Language compulsory? */
633 if (reel_asset->language()) {
634 verify_language_tag (*reel_asset->language(), notes);
640 verify_closed_caption_reel (shared_ptr<const ReelClosedCaptionAsset> reel_asset, vector<VerificationNote>& notes)
642 /* XXX: is Language compulsory? */
643 if (reel_asset->language()) {
644 verify_language_tag (*reel_asset->language(), notes);
651 boost::optional<string> subtitle_language;
657 verify_smpte_subtitle_asset (
658 shared_ptr<const dcp::SMPTESubtitleAsset> asset,
659 vector<VerificationNote>& notes,
663 if (asset->language()) {
664 auto const language = *asset->language();
665 verify_language_tag (language, notes);
666 if (!state.subtitle_language) {
667 state.subtitle_language = language;
668 } else if (state.subtitle_language != language) {
671 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::SUBTITLE_LANGUAGES_DIFFER, *asset->file()
678 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_LANGUAGE, *asset->file()
682 if (boost::filesystem::file_size(*asset->file()) > 115 * 1024 * 1024) {
685 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::TIMED_TEXT_ASSET_TOO_LARGE_IN_BYTES, *asset->file()
689 /* XXX: I'm not sure what Bv2.1_7.2.1 means when it says "the font resource shall not be larger than 10MB"
690 * but I'm hoping that checking for the total size of all fonts being <= 10MB will do.
692 auto fonts = asset->font_data ();
694 for (auto i: fonts) {
695 total_size += i.second.size();
697 if (total_size > 10 * 1024 * 1024) {
700 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::TIMED_TEXT_FONTS_TOO_LARGE_IN_BYTES, *asset->file()
705 if (!asset->start_time()) {
708 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_START_TIME, *asset->file())
710 } else if (asset->start_time() != dcp::Time()) {
713 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::SUBTITLE_START_TIME_NON_ZERO, *asset->file())
720 verify_subtitle_asset (
721 shared_ptr<const SubtitleAsset> asset,
722 function<void (string, optional<boost::filesystem::path>)> stage,
723 boost::filesystem::path xsd_dtd_directory,
724 vector<VerificationNote>& notes,
728 stage ("Checking subtitle XML", asset->file());
729 /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk
730 * gets passed through libdcp which may clean up and therefore hide errors.
732 validate_xml (asset->raw_xml(), xsd_dtd_directory, notes);
734 auto smpte = dynamic_pointer_cast<const SMPTESubtitleAsset>(asset);
736 verify_smpte_subtitle_asset (smpte, notes, state);
742 verify_closed_caption_asset (
743 shared_ptr<const SubtitleAsset> asset,
744 function<void (string, optional<boost::filesystem::path>)> stage,
745 boost::filesystem::path xsd_dtd_directory,
746 vector<VerificationNote>& notes,
750 verify_subtitle_asset (asset, stage, xsd_dtd_directory, notes, state);
752 if (asset->raw_xml().size() > 256 * 1024) {
755 VerificationNote::VERIFY_BV21_ERROR, VerificationNote::CLOSED_CAPTION_XML_TOO_LARGE_IN_BYTES, *asset->file()
765 vector<shared_ptr<dcp::Reel>> reels,
766 optional<int> picture_frame_rate,
767 vector<VerificationNote>& notes,
768 std::function<string (shared_ptr<dcp::Reel>)> xml,
769 std::function<int64_t (shared_ptr<dcp::Reel>)> duration
772 /* end of last subtitle (in editable units) */
773 optional<int64_t> last_out;
774 auto too_short = false;
775 auto too_close = false;
776 auto too_early = false;
777 /* current reel start time (in editable units) */
778 int64_t reel_offset = 0;
780 std::function<void (cxml::ConstNodePtr, int, int, bool)> parse;
781 parse = [&parse, &last_out, &too_short, &too_close, &too_early, &reel_offset](cxml::ConstNodePtr node, int tcr, int pfr, bool first_reel) {
782 if (node->name() == "Subtitle") {
783 dcp::Time in (node->string_attribute("TimeIn"), tcr);
784 dcp::Time out (node->string_attribute("TimeOut"), tcr);
785 if (first_reel && in < dcp::Time(0, 0, 4, 0, tcr)) {
788 auto length = out - in;
789 if (length.as_editable_units(pfr) < 15) {
793 /* XXX: this feels dubious - is it really what Bv2.1 means? */
794 auto distance = reel_offset + in.as_editable_units(pfr) - *last_out;
795 if (distance >= 0 && distance < 2) {
799 last_out = reel_offset + out.as_editable_units(pfr);
801 for (auto i: node->node_children()) {
802 parse(i, tcr, pfr, first_reel);
807 for (auto i = 0U; i < reels.size(); ++i) {
808 /* We need to look at <Subtitle> instances in the XML being checked, so we can't use the subtitles
809 * read in by libdcp's parser.
812 auto doc = make_shared<cxml::Document>("SubtitleReel");
813 doc->read_string (xml(reels[i]));
814 auto const tcr = doc->number_child<int>("TimeCodeRate");
815 parse (doc, tcr, picture_frame_rate.get_value_or(24), i == 0);
816 reel_offset += duration(reels[i]);
822 VerificationNote::VERIFY_WARNING, VerificationNote::FIRST_TEXT_TOO_EARLY
830 VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_TOO_SHORT
838 VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_TOO_CLOSE
845 struct LinesCharactersResult
847 bool warning_length_exceeded = false;
848 bool error_length_exceeded = false;
849 bool line_count_exceeded = false;
855 check_text_lines_and_characters (
856 shared_ptr<SubtitleAsset> asset,
859 LinesCharactersResult* result
865 Event (dcp::Time time_, float position_, int characters_)
867 , position (position_)
868 , characters (characters_)
871 Event (dcp::Time time_, shared_ptr<Event> start_)
877 int position; //< position from 0 at top of screen to 100 at bottom
879 shared_ptr<Event> start;
882 vector<shared_ptr<Event>> events;
884 auto position = [](shared_ptr<const SubtitleString> sub) {
885 switch (sub->v_align()) {
887 return lrintf(sub->v_position() * 100);
889 return lrintf((0.5f + sub->v_position()) * 100);
891 return lrintf((1.0f - sub->v_position()) * 100);
897 for (auto j: asset->subtitles()) {
898 auto text = dynamic_pointer_cast<const SubtitleString>(j);
900 auto in = make_shared<Event>(text->in(), position(text), text->text().length());
901 events.push_back(in);
902 events.push_back(make_shared<Event>(text->out(), in));
906 std::sort(events.begin(), events.end(), [](shared_ptr<Event> const& a, shared_ptr<Event>const& b) {
907 return a->time < b->time;
910 map<int, int> current;
911 for (auto i: events) {
912 if (current.size() > 3) {
913 result->line_count_exceeded = true;
915 for (auto j: current) {
916 if (j.second >= warning_length) {
917 result->warning_length_exceeded = true;
919 if (j.second >= error_length) {
920 result->error_length_exceeded = true;
925 /* end of a subtitle */
926 DCP_ASSERT (current.find(i->start->position) != current.end());
927 if (current[i->start->position] == i->start->characters) {
928 current.erase(i->start->position);
930 current[i->start->position] -= i->start->characters;
933 /* start of a subtitle */
934 if (current.find(i->position) == current.end()) {
935 current[i->position] = i->characters;
937 current[i->position] += i->characters;
946 check_text_timing (vector<shared_ptr<dcp::Reel>> reels, vector<VerificationNote>& notes)
952 optional<int> picture_frame_rate;
953 if (reels[0]->main_picture()) {
954 picture_frame_rate = reels[0]->main_picture()->frame_rate().numerator;
957 if (reels[0]->main_subtitle()) {
958 check_text_timing (reels, picture_frame_rate, notes,
959 [](shared_ptr<dcp::Reel> reel) {
960 return reel->main_subtitle()->asset()->raw_xml();
962 [](shared_ptr<dcp::Reel> reel) {
963 return reel->main_subtitle()->actual_duration();
968 for (auto i = 0U; i < reels[0]->closed_captions().size(); ++i) {
969 check_text_timing (reels, picture_frame_rate, notes,
970 [i](shared_ptr<dcp::Reel> reel) {
971 return reel->closed_captions()[i]->asset()->raw_xml();
973 [i](shared_ptr<dcp::Reel> reel) {
974 return reel->closed_captions()[i]->actual_duration();
981 vector<VerificationNote>
983 vector<boost::filesystem::path> directories,
984 function<void (string, optional<boost::filesystem::path>)> stage,
985 function<void (float)> progress,
986 boost::filesystem::path xsd_dtd_directory
989 xsd_dtd_directory = boost::filesystem::canonical (xsd_dtd_directory);
991 vector<VerificationNote> notes;
994 vector<shared_ptr<DCP>> dcps;
995 for (auto i: directories) {
996 dcps.push_back (shared_ptr<DCP> (new DCP (i)));
999 for (auto dcp: dcps) {
1000 stage ("Checking DCP", dcp->directory());
1003 } catch (ReadError& e) {
1004 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
1005 } catch (XMLError& e) {
1006 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
1007 } catch (MXFFileError& e) {
1008 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
1009 } catch (cxml::Error& e) {
1010 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
1013 if (dcp->standard() != dcp::SMPTE) {
1014 notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::NOT_SMPTE));
1017 for (auto cpl: dcp->cpls()) {
1018 stage ("Checking CPL", cpl->file());
1019 validate_xml (cpl->file().get(), xsd_dtd_directory, notes);
1021 for (auto const& i: cpl->additional_subtitle_languages()) {
1022 verify_language_tag (i, notes);
1025 if (cpl->release_territory()) {
1026 verify_language_tag (cpl->release_territory().get(), notes);
1029 /* Check that the CPL's hash corresponds to the PKL */
1030 for (auto i: dcp->pkls()) {
1031 optional<string> h = i->hash(cpl->id());
1032 if (h && make_digest(ArrayData(*cpl->file())) != *h) {
1033 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::CPL_HASH_INCORRECT));
1037 for (auto reel: cpl->reels()) {
1038 stage ("Checking reel", optional<boost::filesystem::path>());
1040 for (auto i: reel->assets()) {
1041 if (i->duration() && (i->duration().get() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
1042 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::DURATION_TOO_SMALL, i->id()));
1044 if ((i->intrinsic_duration() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
1045 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INTRINSIC_DURATION_TOO_SMALL, i->id()));
1049 if (reel->main_picture()) {
1050 /* Check reel stuff */
1051 auto const frame_rate = reel->main_picture()->frame_rate();
1052 if (frame_rate.denominator != 1 ||
1053 (frame_rate.numerator != 24 &&
1054 frame_rate.numerator != 25 &&
1055 frame_rate.numerator != 30 &&
1056 frame_rate.numerator != 48 &&
1057 frame_rate.numerator != 50 &&
1058 frame_rate.numerator != 60 &&
1059 frame_rate.numerator != 96)) {
1060 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INVALID_PICTURE_FRAME_RATE));
1063 if (reel->main_picture()->asset_ref().resolved()) {
1064 verify_main_picture_asset (dcp, reel->main_picture(), stage, progress, notes);
1068 if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
1069 verify_main_sound_asset (dcp, reel->main_sound(), stage, progress, notes);
1072 if (reel->main_subtitle()) {
1073 verify_main_subtitle_reel (reel->main_subtitle(), notes);
1074 if (reel->main_subtitle()->asset_ref().resolved()) {
1075 verify_subtitle_asset (reel->main_subtitle()->asset(), stage, xsd_dtd_directory, notes, state);
1079 for (auto i: reel->closed_captions()) {
1080 verify_closed_caption_reel (i, notes);
1081 if (i->asset_ref().resolved()) {
1082 verify_closed_caption_asset (i->asset(), stage, xsd_dtd_directory, notes, state);
1087 if (dcp->standard() == dcp::SMPTE) {
1088 check_text_timing (cpl->reels(), notes);
1090 LinesCharactersResult result;
1091 for (auto reel: cpl->reels()) {
1092 if (reel->main_subtitle() && reel->main_subtitle()->asset()) {
1093 check_text_lines_and_characters (reel->main_subtitle()->asset(), 52, 79, &result);
1097 if (result.line_count_exceeded) {
1098 notes.push_back (VerificationNote(VerificationNote::VERIFY_WARNING, VerificationNote::TOO_MANY_SUBTITLE_LINES));
1100 if (result.error_length_exceeded) {
1101 notes.push_back (VerificationNote(VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_LINE_TOO_LONG));
1102 } else if (result.warning_length_exceeded) {
1103 notes.push_back (VerificationNote(VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_LINE_LONGER_THAN_RECOMMENDED));
1106 result = LinesCharactersResult();
1107 for (auto reel: cpl->reels()) {
1108 for (auto i: reel->closed_captions()) {
1110 check_text_lines_and_characters (i->asset(), 32, 32, &result);
1115 if (result.line_count_exceeded) {
1116 notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::TOO_MANY_CLOSED_CAPTION_LINES));
1118 if (result.error_length_exceeded) {
1119 notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::CLOSED_CAPTION_LINE_TOO_LONG));
1124 for (auto pkl: dcp->pkls()) {
1125 stage ("Checking PKL", pkl->file());
1126 validate_xml (pkl->file().get(), xsd_dtd_directory, notes);
1129 if (dcp->asset_map_path()) {
1130 stage ("Checking ASSETMAP", dcp->asset_map_path().get());
1131 validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
1133 notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::MISSING_ASSETMAP));
1141 dcp::note_to_string (dcp::VerificationNote note)
1143 switch (note.code()) {
1144 case dcp::VerificationNote::GENERAL_READ:
1145 return *note.note();
1146 case dcp::VerificationNote::CPL_HASH_INCORRECT:
1147 return "The hash of the CPL in the PKL does not agree with the CPL file.";
1148 case dcp::VerificationNote::INVALID_PICTURE_FRAME_RATE:
1149 return "The picture in a reel has an invalid frame rate.";
1150 case dcp::VerificationNote::PICTURE_HASH_INCORRECT:
1151 return dcp::String::compose("The hash of the picture asset %1 does not agree with the PKL file.", note.file()->filename());
1152 case dcp::VerificationNote::PKL_CPL_PICTURE_HASHES_DIFFER:
1153 return dcp::String::compose("The PKL and CPL hashes differ for the picture asset %1.", note.file()->filename());
1154 case dcp::VerificationNote::SOUND_HASH_INCORRECT:
1155 return dcp::String::compose("The hash of the sound asset %1 does not agree with the PKL file.", note.file()->filename());
1156 case dcp::VerificationNote::PKL_CPL_SOUND_HASHES_DIFFER:
1157 return dcp::String::compose("The PKL and CPL hashes differ for the sound asset %1.", note.file()->filename());
1158 case dcp::VerificationNote::EMPTY_ASSET_PATH:
1159 return "The asset map contains an empty asset path.";
1160 case dcp::VerificationNote::MISSING_ASSET:
1161 return String::compose("The file for an asset in the asset map cannot be found; missing file is %1.", note.file()->filename());
1162 case dcp::VerificationNote::MISMATCHED_STANDARD:
1163 return "The DCP contains both SMPTE and Interop parts.";
1164 case dcp::VerificationNote::XML_VALIDATION_ERROR:
1165 return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get());
1166 case dcp::VerificationNote::MISSING_ASSETMAP:
1167 return "No ASSETMAP or ASSETMAP.xml was found.";
1168 case dcp::VerificationNote::INTRINSIC_DURATION_TOO_SMALL:
1169 return String::compose("The intrinsic duration of an asset is less than 1 second long: %1", note.note().get());
1170 case dcp::VerificationNote::DURATION_TOO_SMALL:
1171 return String::compose("The duration of an asset is less than 1 second long: %1", note.note().get());
1172 case dcp::VerificationNote::PICTURE_FRAME_TOO_LARGE_IN_BYTES:
1173 return String::compose("The instantaneous bit rate of the picture asset %1 is larger than the limit of 250Mbit/s in at least one place.", note.file()->filename());
1174 case dcp::VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE_IN_BYTES:
1175 return String::compose("The instantaneous bit rate of the picture asset %1 is close to the limit of 250Mbit/s in at least one place.", note.file()->filename());
1176 case dcp::VerificationNote::EXTERNAL_ASSET:
1177 return String::compose("An asset that this DCP refers to is not included in the DCP. It may be a VF. Missing asset is %1.", note.note().get());
1178 case dcp::VerificationNote::NOT_SMPTE:
1179 return "This DCP does not use the SMPTE standard, which is required for Bv2.1 compliance.";
1180 case dcp::VerificationNote::BAD_LANGUAGE:
1181 return String::compose("The DCP specifies a language '%1' which does not conform to the RFC 5646 standard.", note.note().get());
1182 case dcp::VerificationNote::PICTURE_ASSET_INVALID_SIZE_IN_PIXELS:
1183 return String::compose("A picture asset's size (%1) is not one of those allowed by Bv2.1 (2048x858, 1998x1080, 4096x1716 or 3996x2160)", note.note().get());
1184 case dcp::VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_2K:
1185 return String::compose("A picture asset's frame rate (%1) is not one of those allowed for 2K DCPs by Bv2.1 (24, 25 or 48fps)", note.note().get());
1186 case dcp::VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_4K:
1187 return String::compose("A picture asset's frame rate (%1) is not 24fps as required for 4K DCPs by Bv2.1", note.note().get());
1188 case dcp::VerificationNote::PICTURE_ASSET_4K_3D:
1189 return "3D 4K DCPs are not allowed by Bv2.1";
1190 case dcp::VerificationNote::CLOSED_CAPTION_XML_TOO_LARGE_IN_BYTES:
1191 return String::compose("The XML for the closed caption asset %1 is longer than the 256KB maximum required by Bv2.1", note.file()->filename());
1192 case dcp::VerificationNote::TIMED_TEXT_ASSET_TOO_LARGE_IN_BYTES:
1193 return String::compose("The total size of the timed text asset %1 is larger than the 115MB maximum required by Bv2.1", note.file()->filename());
1194 case dcp::VerificationNote::TIMED_TEXT_FONTS_TOO_LARGE_IN_BYTES:
1195 return String::compose("The total size of the fonts in timed text asset %1 is larger than the 10MB maximum required by Bv2.1", note.file()->filename());
1196 case dcp::VerificationNote::MISSING_SUBTITLE_LANGUAGE:
1197 return String::compose("The XML for a SMPTE subtitle asset has no <Language> tag, which is required by Bv2.1", note.file()->filename());
1198 case dcp::VerificationNote::SUBTITLE_LANGUAGES_DIFFER:
1199 return String::compose("Some subtitle assets have different <Language> tags than others", note.file()->filename());
1200 case dcp::VerificationNote::MISSING_SUBTITLE_START_TIME:
1201 return String::compose("The XML for a SMPTE subtitle asset has no <StartTime> tag, which is required by Bv2.1", note.file()->filename());
1202 case dcp::VerificationNote::SUBTITLE_START_TIME_NON_ZERO:
1203 return String::compose("The XML for a SMPTE subtitle asset has a non-zero <StartTime> tag, which is disallowed by Bv2.1", note.file()->filename());
1204 case dcp::VerificationNote::FIRST_TEXT_TOO_EARLY:
1205 return "The first subtitle or closed caption is less than 4 seconds from the start of the DCP.";
1206 case dcp::VerificationNote::SUBTITLE_TOO_SHORT:
1207 return "At least one subtitle is less than the minimum of 15 frames suggested by Bv2.1";
1208 case dcp::VerificationNote::SUBTITLE_TOO_CLOSE:
1209 return "At least one pair of subtitles are separated by less than the the minimum of 2 frames suggested by Bv2.1";
1210 case dcp::VerificationNote::TOO_MANY_SUBTITLE_LINES:
1211 return "There are more than 3 subtitle lines in at least one place in the DCP, which Bv2.1 advises against.";
1212 case dcp::VerificationNote::SUBTITLE_LINE_LONGER_THAN_RECOMMENDED:
1213 return "There are more than 52 characters in at least one subtitle line, which Bv2.1 advises against.";
1214 case dcp::VerificationNote::SUBTITLE_LINE_TOO_LONG:
1215 return "There are more than 79 characters in at least one subtitle line, which Bv2.1 strongly advises against.";
1216 case dcp::VerificationNote::TOO_MANY_CLOSED_CAPTION_LINES:
1217 return "There are more than 3 closed caption lines in at least one place, which is disallowed by Bv2.1";
1218 case dcp::VerificationNote::CLOSED_CAPTION_LINE_TOO_LONG:
1219 return "There are more than 32 characters in at least one closed caption line, which is disallowed by Bv2.1";
1220 case dcp::VerificationNote::INVALID_SOUND_FRAME_RATE:
1221 return "A sound asset has a sampling rate other than 48kHz, which is disallowed by Bv2.1";