src/dcp.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of libdcp.
   5
   6     libdcp is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     libdcp is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with libdcp.  If not, see <http://www.gnu.org/licenses/>.
  18
  19     In addition, as a special exception, the copyright holders give
  20     permission to link the code of portions of this program with the
  21     OpenSSL library under certain conditions as described in each
  22     individual source file, and distribute linked combinations
  23     including the two.
  24
  25     You must obey the GNU General Public License in all respects
  26     for all of the code used other than OpenSSL.  If you modify
  27     file(s) with this exception, you may extend this exception to your
  28     version of the file(s), but you are not obligated to do so.  If you
  29     do not wish to do so, delete this exception statement from your
  30     version.  If you delete this exception statement from all source
  31     files in the program, then also delete it here.
  32 */
  33
  34
  35 /** @file  src/dcp.cc
  36  *  @brief DCP class
  37  */
  38
  39
  40 #include "asset_factory.h"
  41 #include "atmos_asset.h"
  42 #include "certificate_chain.h"
  43 #include "compose.hpp"
  44 #include "cpl.h"
  45 #include "dcp.h"
  46 #include "dcp_assert.h"
  47 #include "decrypted_kdm.h"
  48 #include "decrypted_kdm_key.h"
  49 #include "exceptions.h"
  50 #include "filesystem.h"
  51 #include "font_asset.h"
  52 #include "interop_subtitle_asset.h"
  53 #include "metadata.h"
  54 #include "mono_picture_asset.h"
  55 #include "picture_asset.h"
  56 #include "pkl.h"
  57 #include "raw_convert.h"
  58 #include "reel_asset.h"
  59 #include "reel_subtitle_asset.h"
  60 #include "smpte_subtitle_asset.h"
  61 #include "sound_asset.h"
  62 #include "stereo_picture_asset.h"
  63 #include "util.h"
  64 #include "verify.h"
  65 #include "warnings.h"
  66 LIBDCP_DISABLE_WARNINGS
  67 #include <asdcp/AS_DCP.h>
  68 LIBDCP_ENABLE_WARNINGS
  69 #include <xmlsec/xmldsig.h>
  70 #include <xmlsec/app.h>
  71 LIBDCP_DISABLE_WARNINGS
  72 #include <libxml++/libxml++.h>
  73 LIBDCP_ENABLE_WARNINGS
  74 #include <boost/algorithm/string.hpp>
  75 #include <numeric>
  76
  77
  78 using std::cerr;
  79 using std::cout;
  80 using std::dynamic_pointer_cast;
  81 using std::exception;
  82 using std::list;
  83 using std::make_pair;
  84 using std::make_shared;
  85 using std::map;
  86 using std::shared_ptr;
  87 using std::string;
  88 using std::vector;
  89 using boost::algorithm::starts_with;
  90 using boost::optional;
  91 using namespace dcp;
  92
  93
  94 static string const volindex_interop_ns = "http://www.digicine.com/PROTO-ASDCP-VL-20040311#";
  95 static string const volindex_smpte_ns   = "http://www.smpte-ra.org/schemas/429-9/2007/AM";
  96
  97
  98 DCP::DCP (boost::filesystem::path directory)
  99         : _directory (directory)
 100 {
 101         if (!filesystem::exists(directory)) {
 102                 filesystem::create_directories(directory);
 103         }
 104
 105         _directory = filesystem::canonical(_directory);
 106 }
 107
 108
 109 DCP::DCP(DCP&& other)
 110         : _directory(std::move(other._directory))
 111         , _cpls(std::move(other._cpls))
 112         , _pkls(std::move(other._pkls))
 113         , _asset_map(std::move(other._asset_map))
 114         , _new_issuer(std::move(other._new_issuer))
 115         , _new_creator(std::move(other._new_creator))
 116         , _new_issue_date(std::move(other._new_issue_date))
 117         , _new_annotation_text(std::move(other._new_annotation_text))
 118 {
 119
 120 }
 121
 122
 123 DCP&
 124 DCP::operator=(DCP&& other)
 125 {
 126         _directory = std::move(other._directory);
 127         _cpls = std::move(other._cpls);
 128         _pkls = std::move(other._pkls);
 129         _asset_map = std::move(other._asset_map);
 130         _new_issuer = std::move(other._new_issuer);
 131         _new_creator = std::move(other._new_creator);
 132         _new_issue_date = std::move(other._new_issue_date);
 133         _new_annotation_text = std::move(other._new_annotation_text);
 134         return *this;
 135 }
 136
 137
 138 void
 139 DCP::read (vector<dcp::VerificationNote>* notes, bool ignore_incorrect_picture_mxf_type)
 140 {
 141         /* Read the ASSETMAP and PKL */
 142
 143         boost::filesystem::path asset_map_path;
 144         if (filesystem::exists(_directory / "ASSETMAP")) {
 145                 asset_map_path = _directory / "ASSETMAP";
 146         } else if (filesystem::exists(_directory / "ASSETMAP.xml")) {
 147                 asset_map_path = _directory / "ASSETMAP.xml";
 148         } else {
 149                 boost::throw_exception(MissingAssetmapError(_directory));
 150         }
 151
 152         _asset_map = AssetMap(asset_map_path);
 153         auto const pkl_paths = _asset_map->pkl_paths();
 154         auto const standard = _asset_map->standard();
 155
 156         if (pkl_paths.empty()) {
 157                 boost::throw_exception (XMLError ("No packing lists found in asset map"));
 158         }
 159
 160         for (auto i: pkl_paths) {
 161                 _pkls.push_back(make_shared<PKL>(i));
 162         }
 163
 164         /* Now we have:
 165              paths - map of files in the DCP that are not PKLs; key is ID, value is path.
 166              _pkls - PKL objects for each PKL.
 167
 168            Read all the assets from the asset map.
 169          */
 170
 171         /* Make a list of non-CPL/PKL assets so that we can resolve the references
 172            from the CPLs.
 173         */
 174         vector<shared_ptr<Asset>> other_assets;
 175
 176         auto ids_and_paths = _asset_map->asset_ids_and_paths();
 177         for (auto id_and_path: ids_and_paths) {
 178                 auto const id = id_and_path.first;
 179                 auto const path = id_and_path.second;
 180
 181                 if (path == _directory) {
 182                         /* I can't see how this is valid, but it's
 183                            been seen in the wild with a DCP that
 184                            claims to come from ClipsterDCI 5.10.0.5.
 185                         */
 186                         if (notes) {
 187                                 notes->push_back ({VerificationNote::Type::WARNING, VerificationNote::Code::EMPTY_ASSET_PATH});
 188                         }
 189                         continue;
 190                 }
 191
 192                 if (!filesystem::exists(path)) {
 193                         if (notes) {
 194                                 notes->push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::MISSING_ASSET, path});
 195                         }
 196                         continue;
 197                 }
 198
 199                 /* Find the <Type> for this asset from the PKL that contains the asset */
 200                 optional<string> pkl_type;
 201                 for (auto j: _pkls) {
 202                         pkl_type = j->type(id);
 203                         if (pkl_type) {
 204                                 break;
 205                         }
 206                 }
 207
 208                 if (!pkl_type) {
 209                         /* This asset is in the ASSETMAP but not mentioned in any PKL so we don't
 210                          * need to worry about it.
 211                          */
 212                         continue;
 213                 }
 214
 215                 auto remove_parameters = [](string const& n) {
 216                         return n.substr(0, n.find(";"));
 217                 };
 218
 219                 /* Remove any optional parameters (after ;) */
 220                 pkl_type = pkl_type->substr(0, pkl_type->find(";"));
 221
 222                 if (
 223                         pkl_type == remove_parameters(CPL::static_pkl_type(standard)) ||
 224                         pkl_type == remove_parameters(InteropSubtitleAsset::static_pkl_type(standard))) {
 225                         auto p = new xmlpp::DomParser;
 226                         try {
 227                                 p->parse_file(dcp::filesystem::fix_long_path(path).string());
 228                         } catch (std::exception& e) {
 229                                 delete p;
 230                                 throw ReadError(String::compose("XML error in %1", path.string()), e.what());
 231                         }
 232
 233                         auto const root = p->get_document()->get_root_node()->get_name();
 234                         delete p;
 235
 236                         if (root == "CompositionPlaylist") {
 237                                 auto cpl = make_shared<CPL>(path);
 238                                 if (cpl->standard() != standard && notes) {
 239                                         notes->push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_STANDARD});
 240                                 }
 241                                 _cpls.push_back (cpl);
 242                         } else if (root == "DCSubtitle") {
 243                                 if (standard == Standard::SMPTE && notes) {
 244                                         notes->push_back (VerificationNote(VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_STANDARD));
 245                                 }
 246                                 other_assets.push_back (make_shared<InteropSubtitleAsset>(path));
 247                         }
 248                 } else if (
 249                         *pkl_type == remove_parameters(PictureAsset::static_pkl_type(standard)) ||
 250                         *pkl_type == remove_parameters(SoundAsset::static_pkl_type(standard)) ||
 251                         *pkl_type == remove_parameters(AtmosAsset::static_pkl_type(standard)) ||
 252                         *pkl_type == remove_parameters(SMPTESubtitleAsset::static_pkl_type(standard))
 253                         ) {
 254
 255                         bool found_threed_marked_as_twod = false;
 256                         auto asset = asset_factory(path, ignore_incorrect_picture_mxf_type, &found_threed_marked_as_twod);
 257                         if (asset->id() != id) {
 258                                 notes->push_back(VerificationNote(VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_ASSET_MAP_ID).set_id(id).set_other_id(asset->id()));
 259                         }
 260                         other_assets.push_back(asset);
 261                         if (found_threed_marked_as_twod && notes) {
 262                                 notes->push_back ({VerificationNote::Type::WARNING, VerificationNote::Code::THREED_ASSET_MARKED_AS_TWOD, path});
 263                         }
 264                 } else if (*pkl_type == remove_parameters(FontAsset::static_pkl_type(standard))) {
 265                         other_assets.push_back(make_shared<FontAsset>(id, path));
 266                 } else if (*pkl_type == "image/png") {
 267                         /* It's an Interop PNG subtitle; let it go */
 268                 } else {
 269                         throw ReadError (String::compose("Unknown asset type %1 in PKL", *pkl_type));
 270                 }
 271         }
 272
 273         /* Set hashes for assets where we have an idea of what the hash should be in either a CPL or PKL.
 274          * This means that when the hash is later read from these objects the result will be the one that
 275          * it should be, rather the one that it currently is.  This should prevent errors being concealed
 276          * when an asset is corrupted - the hash from the CPL/PKL will disagree with the actual hash of the
 277          * file, revealing the problem.
 278          */
 279
 280         auto hash_from_pkl = [this](string id) -> optional<string> {
 281                 for (auto pkl: _pkls) {
 282                         if (auto pkl_hash = pkl->hash(id)) {
 283                                 return pkl_hash;
 284                         }
 285                 }
 286
 287                 return {};
 288         };
 289
 290         auto hash_from_cpl_or_pkl = [this, &hash_from_pkl](string id) -> optional<string> {
 291                 for (auto cpl: cpls()) {
 292                         for (auto reel_file_asset: cpl->reel_file_assets()) {
 293                                 if (reel_file_asset->asset_ref().id() == id && reel_file_asset->hash()) {
 294                                         return reel_file_asset->hash();
 295                                 }
 296                         }
 297                 }
 298
 299                 return hash_from_pkl(id);
 300         };
 301
 302         for (auto asset: other_assets) {
 303                 if (auto hash = hash_from_cpl_or_pkl(asset->id())) {
 304                         asset->set_hash(*hash);
 305                 }
 306         }
 307
 308         for (auto cpl: cpls()) {
 309                 if (auto hash = hash_from_pkl(cpl->id())) {
 310                         cpl->set_hash(*hash);
 311                 }
 312         }
 313
 314         /* Resolve references */
 315         resolve_refs (other_assets);
 316
 317         /* While we've got the ASSETMAP lets look and see if this DCP refers to things that are not in its ASSETMAP */
 318         if (notes) {
 319                 for (auto i: cpls()) {
 320                         for (auto j: i->reel_file_assets()) {
 321                                 if (!j->asset_ref().resolved() && ids_and_paths.find(j->asset_ref().id()) == ids_and_paths.end()) {
 322                                         notes->push_back (VerificationNote(VerificationNote::Type::WARNING, VerificationNote::Code::EXTERNAL_ASSET, j->asset_ref().id()));
 323                                 }
 324                         }
 325                 }
 326         }
 327 }
 328
 329
 330 void
 331 DCP::resolve_refs (vector<shared_ptr<Asset>> assets)
 332 {
 333         for (auto i: cpls()) {
 334                 i->resolve_refs (assets);
 335         }
 336 }
 337
 338
 339 bool
 340 DCP::equals(DCP const & other, EqualityOptions const& opt, NoteHandler note) const
 341 {
 342         auto a = cpls ();
 343         auto b = other.cpls ();
 344
 345         if (a.size() != b.size()) {
 346                 note (NoteType::ERROR, String::compose ("CPL counts differ: %1 vs %2", a.size(), b.size()));
 347                 return false;
 348         }
 349
 350         bool r = true;
 351
 352         for (auto i: a) {
 353                 auto j = b.begin();
 354                 while (j != b.end() && !(*j)->equals (i, opt, note)) {
 355                         ++j;
 356                 }
 357
 358                 if (j == b.end ()) {
 359                         r = false;
 360                 }
 361         }
 362
 363         return r;
 364 }
 365
 366
 367 void
 368 DCP::add (shared_ptr<CPL> cpl)
 369 {
 370         _cpls.push_back (cpl);
 371 }
 372
 373
 374 bool
 375 DCP::any_encrypted () const
 376 {
 377         for (auto i: cpls()) {
 378                 if (i->any_encrypted()) {
 379                         return true;
 380                 }
 381         }
 382
 383         return false;
 384 }
 385
 386
 387 bool
 388 DCP::all_encrypted () const
 389 {
 390         for (auto i: cpls()) {
 391                 if (!i->all_encrypted()) {
 392                         return false;
 393                 }
 394         }
 395
 396         return true;
 397 }
 398
 399
 400 void
 401 DCP::add (DecryptedKDM const & kdm)
 402 {
 403         auto keys = kdm.keys();
 404         for (auto cpl: cpls()) {
 405                 if (std::any_of(keys.begin(), keys.end(), [cpl](DecryptedKDMKey const& key) { return key.cpl_id() == cpl->id(); })) {
 406                         cpl->add (kdm);
 407                 }
 408         }
 409 }
 410
 411
 412 /** Write the VOLINDEX file.
 413  *  @param standard DCP standard to use (INTEROP or SMPTE)
 414  */
 415 void
 416 DCP::write_volindex (Standard standard) const
 417 {
 418         auto p = _directory;
 419         switch (standard) {
 420         case Standard::INTEROP:
 421                 p /= "VOLINDEX";
 422                 break;
 423         case Standard::SMPTE:
 424                 p /= "VOLINDEX.xml";
 425                 break;
 426         default:
 427                 DCP_ASSERT (false);
 428         }
 429
 430         xmlpp::Document doc;
 431         xmlpp::Element* root;
 432
 433         switch (standard) {
 434         case Standard::INTEROP:
 435                 root = doc.create_root_node ("VolumeIndex", volindex_interop_ns);
 436                 break;
 437         case Standard::SMPTE:
 438                 root = doc.create_root_node ("VolumeIndex", volindex_smpte_ns);
 439                 break;
 440         default:
 441                 DCP_ASSERT (false);
 442         }
 443
 444         root->add_child("Index")->add_child_text ("1");
 445         doc.write_to_file_formatted(dcp::filesystem::fix_long_path(p).string(), "UTF-8");
 446 }
 447
 448
 449 void
 450 DCP::write_xml(shared_ptr<const CertificateChain> signer, bool include_mca_subdescriptors, NameFormat name_format)
 451 {
 452         if (_cpls.empty()) {
 453                 throw MiscError ("Cannot write DCP with no CPLs.");
 454         }
 455
 456         auto standard = std::accumulate (
 457                 std::next(_cpls.begin()), _cpls.end(), _cpls[0]->standard(),
 458                 [](Standard s, shared_ptr<CPL> c) {
 459                         if (s != c->standard()) {
 460                                 throw MiscError ("Cannot make DCP with mixed Interop and SMPTE CPLs.");
 461                         }
 462                         return s;
 463                 }
 464                 );
 465
 466         for (auto i: cpls()) {
 467                 NameFormat::Map values;
 468                 values['t'] = "cpl";
 469                 i->write_xml(_directory / (name_format.get(values, "_" + i->id() + ".xml")), signer, include_mca_subdescriptors);
 470         }
 471
 472         if (_pkls.empty()) {
 473                 _pkls.push_back(
 474                         make_shared<PKL>(
 475                                 standard,
 476                                 _new_annotation_text.get_value_or(String::compose("Created by libdcp %1", dcp::version)),
 477                                 _new_issue_date.get_value_or(LocalTime().as_string()),
 478                                 _new_issuer.get_value_or(String::compose("libdcp %1", dcp::version)),
 479                                 _new_creator.get_value_or(String::compose("libdcp %1", dcp::version))
 480                                 )
 481                         );
 482         }
 483
 484         auto pkl = _pkls.front();
 485
 486         /* The assets may have changed since we read the PKL, so re-add them */
 487         pkl->clear_assets();
 488         for (auto asset: assets()) {
 489                 asset->add_to_pkl(pkl, _directory);
 490         }
 491
 492         NameFormat::Map values;
 493         values['t'] = "pkl";
 494         auto pkl_path = _directory / name_format.get(values, "_" + pkl->id() + ".xml");
 495         pkl->write_xml (pkl_path, signer);
 496
 497         if (!_asset_map) {
 498                 _asset_map = AssetMap(
 499                         standard,
 500                         _new_annotation_text.get_value_or(String::compose("Created by libdcp %1", dcp::version)),
 501                         _new_issue_date.get_value_or(LocalTime().as_string()),
 502                         _new_issuer.get_value_or(String::compose("libdcp %1", dcp::version)),
 503                         _new_creator.get_value_or(String::compose("libdcp %1", dcp::version))
 504                         );
 505         }
 506
 507         /* The assets may have changed since we read the asset map, so re-add them */
 508         _asset_map->clear_assets();
 509         _asset_map->add_asset(pkl->id(), pkl_path, true);
 510         for (auto asset: assets()) {
 511                 asset->add_to_assetmap(*_asset_map, _directory);
 512         }
 513
 514         _asset_map->write_xml(
 515                 _directory / (standard == Standard::INTEROP ? "ASSETMAP" : "ASSETMAP.xml")
 516                 );
 517
 518         write_volindex (standard);
 519 }
 520
 521
 522 vector<shared_ptr<CPL>>
 523 DCP::cpls () const
 524 {
 525         return _cpls;
 526 }
 527
 528
 529 vector<shared_ptr<Asset>>
 530 DCP::assets (bool ignore_unresolved) const
 531 {
 532         vector<shared_ptr<Asset>> assets;
 533         for (auto i: cpls()) {
 534                 assets.push_back (i);
 535                 for (auto j: i->reel_file_assets()) {
 536                         if (ignore_unresolved && !j->asset_ref().resolved()) {
 537                                 continue;
 538                         }
 539
 540                         auto const id = j->asset_ref().id();
 541                         if (std::find_if(assets.begin(), assets.end(), [id](shared_ptr<Asset> asset) { return asset->id() == id; }) == assets.end()) {
 542                                 auto o = j->asset_ref().asset();
 543                                 assets.push_back (o);
 544                                 /* More Interop special-casing */
 545                                 auto sub = dynamic_pointer_cast<InteropSubtitleAsset>(o);
 546                                 if (sub) {
 547                                         add_to_container(assets, sub->font_assets());
 548                                 }
 549                         }
 550                 }
 551         }
 552
 553         return assets;
 554 }
 555
 556
 557 /** Given a list of files that make up 1 or more DCPs, return the DCP directories */
 558 vector<boost::filesystem::path>
 559 DCP::directories_from_files (vector<boost::filesystem::path> files)
 560 {
 561         vector<boost::filesystem::path> d;
 562         for (auto i: files) {
 563                 if (i.filename() == "ASSETMAP" || i.filename() == "ASSETMAP.xml") {
 564                         d.push_back (i.parent_path ());
 565                 }
 566         }
 567         return d;
 568 }
 569
 570
 571 void
 572 DCP::set_issuer(string issuer)
 573 {
 574         for (auto pkl: _pkls) {
 575                 pkl->set_issuer(issuer);
 576         }
 577         if (_asset_map) {
 578                 _asset_map->set_issuer(issuer);
 579         }
 580         _new_issuer = issuer;
 581 }
 582
 583
 584 void
 585 DCP::set_creator(string creator)
 586 {
 587         for (auto pkl: _pkls) {
 588                 pkl->set_creator(creator);
 589         }
 590         if (_asset_map) {
 591                 _asset_map->set_creator(creator);
 592         }
 593         _new_creator = creator;
 594 }
 595
 596
 597 void
 598 DCP::set_issue_date(string issue_date)
 599 {
 600         for (auto pkl: _pkls) {
 601                 pkl->set_issue_date(issue_date);
 602         }
 603         if (_asset_map) {
 604                 _asset_map->set_issue_date(issue_date);
 605         }
 606         _new_issue_date = issue_date;
 607 }
 608
 609
 610 void
 611 DCP::set_annotation_text(string annotation_text)
 612 {
 613         for (auto pkl: _pkls) {
 614                 pkl->set_annotation_text(annotation_text);
 615         }
 616         if (_asset_map) {
 617                 _asset_map->set_annotation_text(annotation_text);
 618         }
 619         _new_annotation_text = annotation_text;
 620 }
 621