src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "audio_buffers.h"
  23 #include "audio_mapping.h"
  24 #include "compose.hpp"
  25 #include "config.h"
  26 #include "constants.h"
  27 #include "cross.h"
  28 #include "dcp_content_type.h"
  29 #include "dcp_video.h"
  30 #include "dcpomatic_log.h"
  31 #include "film.h"
  32 #include "film_util.h"
  33 #include "job.h"
  34 #include "log.h"
  35 #include "ratio.h"
  36 #include "reel_writer.h"
  37 #include "text_content.h"
  38 #include "util.h"
  39 #include "version.h"
  40 #include "writer.h"
  41 #include <dcp/cpl.h>
  42 #include <dcp/locale_convert.h>
  43 #include <dcp/raw_convert.h>
  44 #include <dcp/reel_closed_caption_asset.h>
  45 #include <dcp/reel_file_asset.h>
  46 #include <dcp/reel_subtitle_asset.h>
  47 #include <cerrno>
  48 #include <cfloat>
  49 #include <set>
  50
  51 #include "i18n.h"
  52
  53
  54 /* OS X strikes again */
  55 #undef set_key
  56
  57
  58 using std::cout;
  59 using std::dynamic_pointer_cast;
  60 using std::make_shared;
  61 using std::max;
  62 using std::min;
  63 using std::shared_ptr;
  64 using std::set;
  65 using std::string;
  66 using std::vector;
  67 using std::weak_ptr;
  68 using boost::optional;
  69 #if BOOST_VERSION >= 106100
  70 using namespace boost::placeholders;
  71 #endif
  72 using dcp::Data;
  73 using dcp::ArrayData;
  74 using namespace dcpomatic;
  75
  76
  77 /** @param weak_job Job to report progress to, or 0.
  78  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  79  */
  80 Writer::Writer(weak_ptr<const Film> weak_film, weak_ptr<Job> weak_job, bool text_only)
  81         : WeakConstFilm (weak_film)
  82         , _job(weak_job)
  83         /* These will be reset to sensible values when J2KEncoder is created */
  84         , _maximum_frames_in_memory (8)
  85         , _maximum_queue_size (8)
  86         , _text_only (text_only)
  87 {
  88         auto job = _job.lock ();
  89
  90         int reel_index = 0;
  91         auto const reels = film()->reels();
  92         for (auto p: reels) {
  93                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  94         }
  95
  96         _last_written.resize (reels.size());
  97
  98         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
  99            and captions arrive to the Writer in sequence.  This is not so for video.
 100         */
 101         _audio_reel = _reels.begin ();
 102         _subtitle_reel = _reels.begin ();
 103         for (auto i: film()->closed_caption_tracks()) {
 104                 _caption_reels[i] = _reels.begin ();
 105         }
 106         _atmos_reel = _reels.begin ();
 107
 108         /* Check that the signer is OK */
 109         string reason;
 110         if (!Config::instance()->signer_chain()->valid(&reason)) {
 111                 throw InvalidSignerError (reason);
 112         }
 113 }
 114
 115
 116 void
 117 Writer::start ()
 118 {
 119         if (!_text_only) {
 120                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 121 #ifdef DCPOMATIC_LINUX
 122                 pthread_setname_np (_thread.native_handle(), "writer");
 123 #endif
 124         }
 125 }
 126
 127
 128 Writer::~Writer ()
 129 {
 130         if (!_text_only) {
 131                 terminate_thread (false);
 132         }
 133 }
 134
 135
 136 /** Pass a video frame to the writer for writing to disk at some point.
 137  *  This method can be called with frames out of order.
 138  *  @param encoded JPEG2000-encoded data.
 139  *  @param frame Frame index within the DCP.
 140  *  @param eyes Eyes that this frame image is for.
 141  */
 142 void
 143 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 144 {
 145         boost::mutex::scoped_lock lock (_state_mutex);
 146
 147         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 148                 /* There are too many full frames in memory; wake the main writer thread and
 149                    wait until it sorts everything out */
 150                 _empty_condition.notify_all ();
 151                 _full_condition.wait (lock);
 152         }
 153
 154         QueueItem qi;
 155         qi.type = QueueItem::Type::FULL;
 156         qi.encoded = encoded;
 157         qi.reel = video_reel (frame);
 158         qi.frame = frame - _reels[qi.reel].start ();
 159
 160         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 161
 162         qi.eyes = eyes;
 163         _queue.push_back(qi);
 164         ++_queued_full_in_memory;
 165
 166         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 167         _empty_condition.notify_all ();
 168 }
 169
 170
 171 bool
 172 Writer::can_repeat (Frame frame) const
 173 {
 174         return frame > _reels[video_reel(frame)].start();
 175 }
 176
 177
 178 /** Repeat the last frame that was written to a reel as a new frame.
 179  *  @param frame Frame index within the DCP of the new (repeated) frame.
 180  *  @param eyes Eyes that this repeated frame image is for.
 181  */
 182 void
 183 Writer::repeat (Frame frame, Eyes eyes)
 184 {
 185         boost::mutex::scoped_lock lock (_state_mutex);
 186
 187         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 188                 /* The queue is too big, and the main writer thread can run and fix it, so
 189                    wake it and wait until it has done.
 190                 */
 191                 _empty_condition.notify_all ();
 192                 _full_condition.wait (lock);
 193         }
 194
 195         QueueItem qi;
 196         qi.type = QueueItem::Type::REPEAT;
 197         qi.reel = video_reel (frame);
 198         qi.frame = frame - _reels[qi.reel].start ();
 199         if (film()->three_d() && eyes == Eyes::BOTH) {
 200                 qi.eyes = Eyes::LEFT;
 201                 _queue.push_back (qi);
 202                 qi.eyes = Eyes::RIGHT;
 203                 _queue.push_back (qi);
 204         } else {
 205                 qi.eyes = eyes;
 206                 _queue.push_back (qi);
 207         }
 208
 209         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 210         _empty_condition.notify_all ();
 211 }
 212
 213
 214 void
 215 Writer::fake_write (Frame frame, Eyes eyes)
 216 {
 217         boost::mutex::scoped_lock lock (_state_mutex);
 218
 219         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 220                 /* The queue is too big, and the main writer thread can run and fix it, so
 221                    wake it and wait until it has done.
 222                 */
 223                 _empty_condition.notify_all ();
 224                 _full_condition.wait (lock);
 225         }
 226
 227         size_t const reel = video_reel (frame);
 228         Frame const frame_in_reel = frame - _reels[reel].start ();
 229
 230         QueueItem qi;
 231         qi.type = QueueItem::Type::FAKE;
 232
 233         {
 234                 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
 235                 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
 236         }
 237
 238         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 239
 240         qi.reel = reel;
 241         qi.frame = frame_in_reel;
 242         qi.eyes = eyes;
 243         _queue.push_back(qi);
 244
 245         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 246         _empty_condition.notify_all ();
 247 }
 248
 249
 250 /** Write some audio frames to the DCP.
 251  *  @param audio Audio data.
 252  *  @param time Time of this data within the DCP.
 253  *  This method is not thread safe.
 254  */
 255 void
 256 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 257 {
 258         DCPOMATIC_ASSERT (audio);
 259
 260         int const afr = film()->audio_frame_rate();
 261
 262         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 263
 264         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 265
 266         DCPTime t = time;
 267         while (t < end) {
 268
 269                 if (_audio_reel == _reels.end ()) {
 270                         /* This audio is off the end of the last reel; ignore it */
 271                         return;
 272                 }
 273
 274                 if (end <= _audio_reel->period().to) {
 275                         /* Easy case: we can write all the audio to this reel */
 276                         _audio_reel->write (audio);
 277                         t = end;
 278                 } else if (_audio_reel->period().to <= t) {
 279                         /* This reel is entirely before the start of our audio; just skip the reel */
 280                         ++_audio_reel;
 281                 } else {
 282                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 283                         DCPTime part_lengths[2] = {
 284                                 _audio_reel->period().to - t,
 285                                 end - _audio_reel->period().to
 286                         };
 287
 288                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 289                         Frame part_frames[2] = {
 290                                 part_lengths[0].frames_ceil(afr),
 291                                 part_lengths[1].frames_floor(afr)
 292                         };
 293
 294                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 295
 296                         if (part_frames[0]) {
 297                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 298                                 _audio_reel->write (part);
 299                         }
 300
 301                         if (part_frames[1]) {
 302                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 303                         } else {
 304                                 audio.reset ();
 305                         }
 306
 307                         ++_audio_reel;
 308                         t += part_lengths[0];
 309                 }
 310         }
 311 }
 312
 313
 314 void
 315 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 316 {
 317         if (_atmos_reel->period().to == time) {
 318                 ++_atmos_reel;
 319                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 320         }
 321
 322         /* We assume that we get a video frame's worth of data here */
 323         _atmos_reel->write (atmos, metadata);
 324 }
 325
 326
 327 /** Caller must hold a lock on _state_mutex */
 328 bool
 329 Writer::have_sequenced_image_at_queue_head ()
 330 {
 331         if (_queue.empty ()) {
 332                 return false;
 333         }
 334
 335         _queue.sort ();
 336         auto const & f = _queue.front();
 337         return _last_written[f.reel].next(f);
 338 }
 339
 340
 341 bool
 342 Writer::LastWritten::next (QueueItem qi) const
 343 {
 344         if (qi.eyes == Eyes::BOTH) {
 345                 /* 2D */
 346                 return qi.frame == (_frame + 1);
 347         }
 348
 349         /* 3D */
 350
 351         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 352                 return true;
 353         }
 354
 355         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 356                 return true;
 357         }
 358
 359         return false;
 360 }
 361
 362
 363 void
 364 Writer::LastWritten::update (QueueItem qi)
 365 {
 366         _frame = qi.frame;
 367         _eyes = qi.eyes;
 368 }
 369
 370
 371 void
 372 Writer::thread ()
 373 try
 374 {
 375         start_of_thread ("Writer");
 376
 377         while (true)
 378         {
 379                 boost::mutex::scoped_lock lock (_state_mutex);
 380
 381                 while (true) {
 382
 383                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 384                                 /* We've got something to do: go and do it */
 385                                 break;
 386                         }
 387
 388                         /* Nothing to do: wait until something happens which may indicate that we do */
 389                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 390                         _empty_condition.wait (lock);
 391                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 392                 }
 393
 394                 /* We stop here if we have been asked to finish, and if either the queue
 395                    is empty or we do not have a sequenced image at its head (if this is the
 396                    case we will never terminate as no new frames will be sent once
 397                    _finish is true).
 398                 */
 399                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 400                         /* (Hopefully temporarily) log anything that was not written */
 401                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 402                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 403                                 for (auto const& i: _queue) {
 404                                         if (i.type == QueueItem::Type::FULL) {
 405                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 406                                         } else {
 407                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 408                                         }
 409                                 }
 410                         }
 411                         return;
 412                 }
 413
 414                 /* Write any frames that we can write; i.e. those that are in sequence. */
 415                 while (have_sequenced_image_at_queue_head ()) {
 416                         auto qi = _queue.front ();
 417                         _last_written[qi.reel].update (qi);
 418                         _queue.pop_front ();
 419                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 420                                 --_queued_full_in_memory;
 421                         }
 422
 423                         lock.unlock ();
 424
 425                         auto& reel = _reels[qi.reel];
 426
 427                         switch (qi.type) {
 428                         case QueueItem::Type::FULL:
 429                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 430                                 if (!qi.encoded) {
 431                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 432                                 }
 433                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 434                                 ++_full_written;
 435                                 break;
 436                         case QueueItem::Type::FAKE:
 437                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 438                                 reel.fake_write (qi.size);
 439                                 ++_fake_written;
 440                                 break;
 441                         case QueueItem::Type::REPEAT:
 442                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 443                                 reel.repeat_write (qi.frame, qi.eyes);
 444                                 ++_repeat_written;
 445                                 break;
 446                         }
 447
 448                         lock.lock ();
 449                         _full_condition.notify_all ();
 450                 }
 451
 452                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 453                         /* Too many frames in memory which can't yet be written to the stream.
 454                            Write some FULL frames to disk.
 455                         */
 456
 457                         /* Find one from the back of the queue */
 458                         _queue.sort ();
 459                         auto item = _queue.rbegin();
 460                         while (item != _queue.rend() && (item->type != QueueItem::Type::FULL || !item->encoded)) {
 461                                 ++item;
 462                         }
 463
 464                         DCPOMATIC_ASSERT(item != _queue.rend());
 465                         ++_pushed_to_disk;
 466                         /* For the log message below */
 467                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 468                         lock.unlock ();
 469
 470                         /* i is valid here, even though we don't hold a lock on the mutex,
 471                            since list iterators are unaffected by insertion and only this
 472                            thread could erase the last item in the list.
 473                         */
 474
 475                         LOG_GENERAL("Writer full; pushes %1 to disk while awaiting %2", item->frame, awaiting);
 476
 477                         item->encoded->write_via_temp(
 478                                 film()->j2c_path(item->reel, item->frame, item->eyes, true),
 479                                 film()->j2c_path(item->reel, item->frame, item->eyes, false)
 480                                 );
 481
 482                         lock.lock ();
 483                         item->encoded.reset();
 484                         --_queued_full_in_memory;
 485                         _full_condition.notify_all ();
 486                 }
 487         }
 488 }
 489 catch (...)
 490 {
 491         store_current ();
 492 }
 493
 494
 495 void
 496 Writer::terminate_thread (bool can_throw)
 497 {
 498         boost::this_thread::disable_interruption dis;
 499
 500         boost::mutex::scoped_lock lock (_state_mutex);
 501
 502         _finish = true;
 503         _empty_condition.notify_all ();
 504         _full_condition.notify_all ();
 505         lock.unlock ();
 506
 507         try {
 508                 _thread.join ();
 509         } catch (...) {}
 510
 511         if (can_throw) {
 512                 rethrow ();
 513         }
 514 }
 515
 516
 517 void
 518 Writer::calculate_digests ()
 519 {
 520         auto job = _job.lock ();
 521         if (job) {
 522                 job->sub (_("Computing digests"));
 523         }
 524
 525         boost::asio::io_service service;
 526         boost::thread_group pool;
 527
 528         auto work = make_shared<boost::asio::io_service::work>(service);
 529
 530         int const threads = max (1, Config::instance()->master_encoding_threads());
 531
 532         for (int i = 0; i < threads; ++i) {
 533                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 534         }
 535
 536         std::function<void (int, int64_t, int64_t)> set_progress;
 537         if (job) {
 538                 set_progress = boost::bind(&Writer::set_digest_progress, this, job.get(), _1, _2, _3);
 539         } else {
 540                 set_progress = [](int, int64_t, int64_t) {
 541                         boost::this_thread::interruption_point();
 542                 };
 543         }
 544
 545         int index = 0;
 546
 547         for (auto& i: _reels) {
 548                 service.post(
 549                         boost::bind(
 550                                 &ReelWriter::calculate_digests,
 551                                 &i,
 552                                 std::function<void (int64_t, int64_t)>(boost::bind(set_progress, index, _1, _2))
 553                                 ));
 554                 ++index;
 555         }
 556         service.post(
 557                 boost::bind(
 558                         &Writer::calculate_referenced_digests,
 559                         this,
 560                         std::function<void (int64_t, int64_t)>(boost::bind(set_progress, index, _1, _2))
 561                         ));
 562
 563         work.reset ();
 564
 565         try {
 566                 pool.join_all ();
 567         } catch (boost::thread_interrupted) {
 568                 /* join_all was interrupted, so we need to interrupt the threads
 569                  * in our pool then try again to join them.
 570                  */
 571                 pool.interrupt_all ();
 572                 pool.join_all ();
 573         }
 574
 575         service.stop ();
 576 }
 577
 578
 579 /** @param output_dcp Path to DCP folder to write */
 580 void
 581 Writer::finish (boost::filesystem::path output_dcp)
 582 {
 583         if (_thread.joinable()) {
 584                 LOG_GENERAL_NC ("Terminating writer thread");
 585                 terminate_thread (true);
 586         }
 587
 588         LOG_GENERAL_NC ("Finishing ReelWriters");
 589
 590         for (auto& reel: _reels) {
 591                 write_hanging_text(reel);
 592                 reel.finish(output_dcp);
 593         }
 594
 595         LOG_GENERAL_NC ("Writing XML");
 596
 597         dcp::DCP dcp (output_dcp);
 598
 599         auto cpl = make_shared<dcp::CPL>(
 600                 film()->dcp_name(),
 601                 film()->dcp_content_type()->libdcp_kind(),
 602                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 603                 );
 604
 605         dcp.add (cpl);
 606
 607         calculate_digests ();
 608
 609         /* Add reels */
 610
 611         for (auto& i: _reels) {
 612                 cpl->add(i.create_reel(_reel_assets, output_dcp, _have_subtitles, _have_closed_captions));
 613         }
 614
 615         /* Add metadata */
 616
 617         auto creator = Config::instance()->dcp_creator();
 618         if (creator.empty()) {
 619                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 620         }
 621
 622         auto issuer = Config::instance()->dcp_issuer();
 623         if (issuer.empty()) {
 624                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 625         }
 626
 627         cpl->set_creator (creator);
 628         cpl->set_issuer (issuer);
 629
 630         cpl->set_ratings (film()->ratings());
 631
 632         vector<dcp::ContentVersion> cv;
 633         for (auto i: film()->content_versions()) {
 634                 /* Make sure we don't end up writing an empty <LabelText> node as some validators
 635                  * complain about that.
 636                  */
 637                 cv.push_back(!i.empty() ? dcp::ContentVersion(i) : dcp::ContentVersion("1"));
 638         }
 639         if (cv.empty()) {
 640                 cv = { dcp::ContentVersion("1") };
 641         }
 642         cpl->set_content_versions (cv);
 643
 644         cpl->set_full_content_title_text (film()->name());
 645         cpl->set_full_content_title_text_language (film()->name_language());
 646         if (film()->release_territory()) {
 647                 cpl->set_release_territory (*film()->release_territory());
 648         }
 649         cpl->set_version_number (film()->version_number());
 650         cpl->set_status (film()->status());
 651         if (film()->chain()) {
 652                 cpl->set_chain (*film()->chain());
 653         }
 654         if (film()->distributor()) {
 655                 cpl->set_distributor (*film()->distributor());
 656         }
 657         if (film()->facility()) {
 658                 cpl->set_facility (*film()->facility());
 659         }
 660         if (film()->luminance()) {
 661                 cpl->set_luminance (*film()->luminance());
 662         }
 663         if (film()->sign_language_video_language()) {
 664                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 665         }
 666
 667         dcp::MCASoundField field;
 668         if (channel_is_mapped(film(), dcp::Channel::BSL) || channel_is_mapped(film(), dcp::Channel::BSR)) {
 669                 field = dcp::MCASoundField::SEVEN_POINT_ONE;
 670         } else {
 671                 field = dcp::MCASoundField::FIVE_POINT_ONE;
 672         }
 673
 674         auto const audio_channels = film()->audio_channels();
 675         dcp::MainSoundConfiguration msc(field, audio_channels);
 676         for (auto i: film()->mapped_audio_channels()) {
 677                 if (i < audio_channels) {
 678                         msc.set_mapping(i, static_cast<dcp::Channel>(i));
 679                 }
 680         }
 681
 682         cpl->set_main_sound_configuration(msc);
 683         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 684         cpl->set_main_picture_stored_area (film()->frame_size());
 685
 686         auto active_area = film()->active_area();
 687         if (active_area.width > 0 && active_area.height > 0) {
 688                 /* It's not allowed to have a zero active area width or height, and the sizes must be multiples of 2 */
 689                 cpl->set_main_picture_active_area({ active_area.width & ~1, active_area.height & ~1});
 690         }
 691
 692         auto sl = film()->subtitle_languages().second;
 693         if (!sl.empty()) {
 694                 cpl->set_additional_subtitle_languages(sl);
 695         }
 696
 697         auto signer = Config::instance()->signer_chain();
 698         /* We did check earlier, but check again here to be on the safe side */
 699         string reason;
 700         if (!signer->valid (&reason)) {
 701                 throw InvalidSignerError (reason);
 702         }
 703
 704         dcp.set_issuer(issuer);
 705         dcp.set_creator(creator);
 706         dcp.set_annotation_text(film()->dcp_name());
 707
 708         dcp.write_xml(signer, !film()->limit_to_smpte_bv20(), Config::instance()->dcp_metadata_filename_format());
 709
 710         LOG_GENERAL (
 711                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 712                 );
 713
 714         write_cover_sheet (output_dcp);
 715 }
 716
 717
 718 void
 719 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 720 {
 721         auto const cover = film()->file("COVER_SHEET.txt");
 722         dcp::File file(cover, "w");
 723         if (!file) {
 724                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 725         }
 726
 727         auto text = Config::instance()->cover_sheet ();
 728         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 729         auto cpls = film()->cpls();
 730         if (!cpls.empty()) {
 731                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 732         }
 733         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 734         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 735
 736         auto audio_language = film()->audio_language();
 737         if (audio_language) {
 738                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 739         } else {
 740                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 741         }
 742
 743         auto subtitle_languages = film()->subtitle_languages();
 744         if (subtitle_languages.first) {
 745                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 746         } else {
 747                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 748         }
 749
 750         boost::uintmax_t size = 0;
 751         for (
 752                 auto i = dcp::filesystem::recursive_directory_iterator(output_dcp);
 753                 i != dcp::filesystem::recursive_directory_iterator();
 754                 ++i) {
 755                 if (dcp::filesystem::is_regular_file(i->path())) {
 756                         size += dcp::filesystem::file_size(i->path());
 757                 }
 758         }
 759
 760         if (size > (1000000000L)) {
 761                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 762         } else {
 763                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 764         }
 765
 766         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 767         auto description = String::compose("%1.%2", ch.first, ch.second);
 768
 769         if (description == "0.0") {
 770                 description = _("None");
 771         } else if (description == "1.0") {
 772                 description = _("Mono");
 773         } else if (description == "2.0") {
 774                 description = _("Stereo");
 775         }
 776         boost::algorithm::replace_all (text, "$AUDIO", description);
 777
 778         auto const hmsf = film()->length().split(film()->video_frame_rate());
 779         string length;
 780         if (hmsf.h == 0 && hmsf.m == 0) {
 781                 length = String::compose("%1s", hmsf.s);
 782         } else if (hmsf.h == 0 && hmsf.m > 0) {
 783                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 784         } else if (hmsf.h > 0 && hmsf.m > 0) {
 785                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 786         }
 787
 788         boost::algorithm::replace_all (text, "$LENGTH", length);
 789
 790         file.checked_write(text.c_str(), text.length());
 791 }
 792
 793
 794 /** @param frame Frame index within the whole DCP.
 795  *  @return true if we can fake-write this frame.
 796  */
 797 bool
 798 Writer::can_fake_write (Frame frame) const
 799 {
 800         if (film()->encrypted()) {
 801                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 802                 return false;
 803         }
 804
 805         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 806            parameters in the asset writer.
 807         */
 808
 809         auto const & reel = _reels[video_reel(frame)];
 810
 811         /* Make frame relative to the start of the reel */
 812         frame -= reel.start ();
 813         return (frame != 0 && frame < reel.first_nonexistent_frame());
 814 }
 815
 816
 817 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 818 void
 819 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 820 {
 821         vector<ReelWriter>::iterator* reel = nullptr;
 822
 823         switch (type) {
 824         case TextType::OPEN_SUBTITLE:
 825                 reel = &_subtitle_reel;
 826                 _have_subtitles = true;
 827                 break;
 828         case TextType::CLOSED_CAPTION:
 829                 DCPOMATIC_ASSERT (track);
 830                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 831                 reel = &_caption_reels[*track];
 832                 _have_closed_captions.insert (*track);
 833                 break;
 834         default:
 835                 DCPOMATIC_ASSERT (false);
 836         }
 837
 838         DCPOMATIC_ASSERT (*reel != _reels.end());
 839         while ((*reel)->period().to <= period.from) {
 840                 ++(*reel);
 841                 DCPOMATIC_ASSERT (*reel != _reels.end());
 842                 write_hanging_text (**reel);
 843         }
 844
 845         auto back_off = [this](DCPTimePeriod period) {
 846                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 847                 return period;
 848         };
 849
 850         if (period.to > (*reel)->period().to) {
 851                 /* This text goes off the end of the reel.  Store parts of it that should go into
 852                  * other reels.
 853                  */
 854                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 855                         auto overlap = i->period().overlap(period);
 856                         if (overlap) {
 857                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 858                         }
 859                 }
 860                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 861                  * for subtitles being too close together.
 862                  */
 863                 period.to = (*reel)->period().to;
 864                 period = back_off(period);
 865         }
 866
 867         (*reel)->write(text, type, track, period, _fonts, _chosen_interop_font);
 868 }
 869
 870
 871 void
 872 Writer::write (vector<shared_ptr<Font>> fonts)
 873 {
 874         if (fonts.empty()) {
 875                 return;
 876         }
 877
 878         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 879         auto fix_id = [](string id) {
 880                 return id.empty() ? "font" : id;
 881         };
 882
 883         if (film()->interop()) {
 884                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 885                  * even write them as they upset some validators.  Set up _fonts so that every
 886                  * font used by any subtitle will be written with the same ID.
 887                  */
 888                 for (size_t i = 0; i < fonts.size(); ++i) {
 889                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 890                 }
 891                 _chosen_interop_font = fonts[0];
 892         } else {
 893                 for (auto font: fonts) {
 894                         _fonts.put(font, fix_id(font->id()));
 895                 }
 896         }
 897 }
 898
 899
 900 bool
 901 operator< (QueueItem const & a, QueueItem const & b)
 902 {
 903         if (a.reel != b.reel) {
 904                 return a.reel < b.reel;
 905         }
 906
 907         if (a.frame != b.frame) {
 908                 return a.frame < b.frame;
 909         }
 910
 911         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 912 }
 913
 914
 915 bool
 916 operator== (QueueItem const & a, QueueItem const & b)
 917 {
 918         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 919 }
 920
 921
 922 void
 923 Writer::set_encoder_threads (int threads)
 924 {
 925         boost::mutex::scoped_lock lm (_state_mutex);
 926         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 927         _maximum_queue_size = threads * 16;
 928 }
 929
 930
 931 void
 932 Writer::write (ReferencedReelAsset asset)
 933 {
 934         _reel_assets.push_back (asset);
 935
 936         if (dynamic_pointer_cast<dcp::ReelSubtitleAsset>(asset.asset)) {
 937                 _have_subtitles = true;
 938         } else if (auto ccap = dynamic_pointer_cast<dcp::ReelClosedCaptionAsset>(asset.asset)) {
 939                 /* This feels quite fragile. We have a referenced reel and want to know if it's
 940                  * part of a given closed-caption track so that we can fill if it has any
 941                  * missing reels.  I guess for that purpose almost any DCPTextTrack values are
 942                  * fine so long as they are consistent.
 943                  */
 944                 DCPTextTrack track;
 945                 track.name = ccap->annotation_text().get_value_or("");
 946                 track.language = dcp::LanguageTag(ccap->language().get_value_or("en-US"));
 947                 if (_have_closed_captions.find(track) == _have_closed_captions.end()) {
 948                         _have_closed_captions.insert(track);
 949                 }
 950         }
 951 }
 952
 953
 954 size_t
 955 Writer::video_reel (int frame) const
 956 {
 957         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 958         size_t reel_index = 0;
 959         while (reel_index < _reels.size() && !_reels[reel_index].period().contains(t)) {
 960                 ++reel_index;
 961         }
 962
 963         DCPOMATIC_ASSERT(reel_index < _reels.size ());
 964         return reel_index;
 965 }
 966
 967
 968 /** Update job progress with information about the progress of a single digest calculation
 969  *  thread.
 970  *  @param id Unique identifier for the thread whose progress has changed.
 971  *  @param done Number of bytes that this thread has processed.
 972  *  @param size Total number of bytes that this thread must process.
 973  */
 974 void
 975 Writer::set_digest_progress(Job* job, int id, int64_t done, int64_t size)
 976 {
 977         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 978
 979         /* Update the progress for this thread */
 980         _digest_progresses[id] = std::make_pair(done, size);
 981
 982         /* Get the total progress across all threads and use it to set job progress */
 983         int64_t total_done = 0;
 984         int64_t total_size = 0;
 985         for (auto const& i: _digest_progresses) {
 986                 total_done += i.second.first;
 987                 total_size += i.second.second;
 988         }
 989
 990         job->set_progress(float(total_done) / total_size);
 991
 992         Waker waker;
 993         waker.nudge ();
 994
 995         boost::this_thread::interruption_point();
 996 }
 997
 998
 999 /** Calculate hashes for any referenced MXF assets which do not already have one */
1000 void
1001 Writer::calculate_referenced_digests(std::function<void (int64_t, int64_t)> set_progress)
1002 try
1003 {
1004         int64_t total_size = 0;
1005         for (auto const& i: _reel_assets) {
1006                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1007                 if (file && !file->hash()) {
1008                         auto filename = file->asset_ref().asset()->file();
1009                         DCPOMATIC_ASSERT(filename);
1010                         total_size += boost::filesystem::file_size(*filename);
1011                 }
1012         }
1013
1014         int64_t total_done = 0;
1015         for (auto const& i: _reel_assets) {
1016                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1017                 if (file && !file->hash()) {
1018                         file->asset_ref().asset()->hash([&total_done, total_size, set_progress](int64_t done, int64_t) {
1019                                 set_progress(total_done + done, total_size);
1020                         });
1021                         total_done += boost::filesystem::file_size(*file->asset_ref().asset()->file());
1022                         file->set_hash (file->asset_ref().asset()->hash());
1023                 }
1024         }
1025 } catch (boost::thread_interrupted) {
1026         /* set_progress contains an interruption_point, so any of these methods
1027          * may throw thread_interrupted, at which point we just give up.
1028          */
1029 }
1030
1031
1032 void
1033 Writer::write_hanging_text (ReelWriter& reel)
1034 {
1035         vector<HangingText> new_hanging_texts;
1036         for (auto i: _hanging_texts) {
1037                 if (i.period.from == reel.period().from) {
1038                         reel.write(i.text, i.type, i.track, i.period, _fonts, _chosen_interop_font);
1039                 } else {
1040                         new_hanging_texts.push_back (i);
1041                 }
1042         }
1043         _hanging_texts = new_hanging_texts;
1044 }