src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "writer.h"
  23 #include "compose.hpp"
  24 #include "film.h"
  25 #include "ratio.h"
  26 #include "log.h"
  27 #include "dcpomatic_log.h"
  28 #include "dcp_video.h"
  29 #include "dcp_content_type.h"
  30 #include "audio_mapping.h"
  31 #include "config.h"
  32 #include "job.h"
  33 #include "cross.h"
  34 #include "audio_buffers.h"
  35 #include "version.h"
  36 #include "util.h"
  37 #include "reel_writer.h"
  38 #include "text_content.h"
  39 #include <dcp/cpl.h>
  40 #include <dcp/locale_convert.h>
  41 #include <dcp/reel_file_asset.h>
  42 #include <fstream>
  43 #include <cerrno>
  44 #include <iostream>
  45 #include <cfloat>
  46 #include <set>
  47
  48 #include "i18n.h"
  49
  50
  51 /* OS X strikes again */
  52 #undef set_key
  53
  54
  55 using std::cout;
  56 using std::dynamic_pointer_cast;
  57 using std::make_shared;
  58 using std::max;
  59 using std::min;
  60 using std::shared_ptr;
  61 using std::set;
  62 using std::string;
  63 using std::vector;
  64 using std::weak_ptr;
  65 using boost::optional;
  66 #if BOOST_VERSION >= 106100
  67 using namespace boost::placeholders;
  68 #endif
  69 using dcp::Data;
  70 using dcp::ArrayData;
  71 using namespace dcpomatic;
  72
  73
  74 /** @param j Job to report progress to, or 0.
  75  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  76  */
  77 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
  78         : WeakConstFilm (weak_film)
  79         , _job (j)
  80         /* These will be reset to sensible values when J2KEncoder is created */
  81         , _maximum_frames_in_memory (8)
  82         , _maximum_queue_size (8)
  83         , _text_only (text_only)
  84 {
  85         auto job = _job.lock ();
  86
  87         int reel_index = 0;
  88         auto const reels = film()->reels();
  89         for (auto p: reels) {
  90                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  91         }
  92
  93         _last_written.resize (reels.size());
  94
  95         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
  96            and captions arrive to the Writer in sequence.  This is not so for video.
  97         */
  98         _audio_reel = _reels.begin ();
  99         _subtitle_reel = _reels.begin ();
 100         for (auto i: film()->closed_caption_tracks()) {
 101                 _caption_reels[i] = _reels.begin ();
 102         }
 103         _atmos_reel = _reels.begin ();
 104
 105         /* Check that the signer is OK */
 106         string reason;
 107         if (!Config::instance()->signer_chain()->valid(&reason)) {
 108                 throw InvalidSignerError (reason);
 109         }
 110 }
 111
 112
 113 void
 114 Writer::start ()
 115 {
 116         if (!_text_only) {
 117                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 118 #ifdef DCPOMATIC_LINUX
 119                 pthread_setname_np (_thread.native_handle(), "writer");
 120 #endif
 121         }
 122 }
 123
 124
 125 Writer::~Writer ()
 126 {
 127         if (!_text_only) {
 128                 terminate_thread (false);
 129         }
 130 }
 131
 132
 133 /** Pass a video frame to the writer for writing to disk at some point.
 134  *  This method can be called with frames out of order.
 135  *  @param encoded JPEG2000-encoded data.
 136  *  @param frame Frame index within the DCP.
 137  *  @param eyes Eyes that this frame image is for.
 138  */
 139 void
 140 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 141 {
 142         boost::mutex::scoped_lock lock (_state_mutex);
 143
 144         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 145                 /* There are too many full frames in memory; wake the main writer thread and
 146                    wait until it sorts everything out */
 147                 _empty_condition.notify_all ();
 148                 _full_condition.wait (lock);
 149         }
 150
 151         QueueItem qi;
 152         qi.type = QueueItem::Type::FULL;
 153         qi.encoded = encoded;
 154         qi.reel = video_reel (frame);
 155         qi.frame = frame - _reels[qi.reel].start ();
 156
 157         if (film()->three_d() && eyes == Eyes::BOTH) {
 158                 /* 2D material in a 3D DCP; fake the 3D */
 159                 qi.eyes = Eyes::LEFT;
 160                 _queue.push_back (qi);
 161                 ++_queued_full_in_memory;
 162                 qi.eyes = Eyes::RIGHT;
 163                 _queue.push_back (qi);
 164                 ++_queued_full_in_memory;
 165         } else {
 166                 qi.eyes = eyes;
 167                 _queue.push_back (qi);
 168                 ++_queued_full_in_memory;
 169         }
 170
 171         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 172         _empty_condition.notify_all ();
 173 }
 174
 175
 176 bool
 177 Writer::can_repeat (Frame frame) const
 178 {
 179         return frame > _reels[video_reel(frame)].start();
 180 }
 181
 182
 183 /** Repeat the last frame that was written to a reel as a new frame.
 184  *  @param frame Frame index within the DCP of the new (repeated) frame.
 185  *  @param eyes Eyes that this repeated frame image is for.
 186  */
 187 void
 188 Writer::repeat (Frame frame, Eyes eyes)
 189 {
 190         boost::mutex::scoped_lock lock (_state_mutex);
 191
 192         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 193                 /* The queue is too big, and the main writer thread can run and fix it, so
 194                    wake it and wait until it has done.
 195                 */
 196                 _empty_condition.notify_all ();
 197                 _full_condition.wait (lock);
 198         }
 199
 200         QueueItem qi;
 201         qi.type = QueueItem::Type::REPEAT;
 202         qi.reel = video_reel (frame);
 203         qi.frame = frame - _reels[qi.reel].start ();
 204         if (film()->three_d() && eyes == Eyes::BOTH) {
 205                 qi.eyes = Eyes::LEFT;
 206                 _queue.push_back (qi);
 207                 qi.eyes = Eyes::RIGHT;
 208                 _queue.push_back (qi);
 209         } else {
 210                 qi.eyes = eyes;
 211                 _queue.push_back (qi);
 212         }
 213
 214         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 215         _empty_condition.notify_all ();
 216 }
 217
 218
 219 void
 220 Writer::fake_write (Frame frame, Eyes eyes)
 221 {
 222         boost::mutex::scoped_lock lock (_state_mutex);
 223
 224         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 225                 /* The queue is too big, and the main writer thread can run and fix it, so
 226                    wake it and wait until it has done.
 227                 */
 228                 _empty_condition.notify_all ();
 229                 _full_condition.wait (lock);
 230         }
 231
 232         size_t const reel = video_reel (frame);
 233         Frame const frame_in_reel = frame - _reels[reel].start ();
 234
 235         QueueItem qi;
 236         qi.type = QueueItem::Type::FAKE;
 237
 238         {
 239                 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
 240                 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
 241         }
 242
 243         qi.reel = reel;
 244         qi.frame = frame_in_reel;
 245         if (film()->three_d() && eyes == Eyes::BOTH) {
 246                 qi.eyes = Eyes::LEFT;
 247                 _queue.push_back (qi);
 248                 qi.eyes = Eyes::RIGHT;
 249                 _queue.push_back (qi);
 250         } else {
 251                 qi.eyes = eyes;
 252                 _queue.push_back (qi);
 253         }
 254
 255         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 256         _empty_condition.notify_all ();
 257 }
 258
 259
 260 /** Write some audio frames to the DCP.
 261  *  @param audio Audio data.
 262  *  @param time Time of this data within the DCP.
 263  *  This method is not thread safe.
 264  */
 265 void
 266 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 267 {
 268         DCPOMATIC_ASSERT (audio);
 269
 270         int const afr = film()->audio_frame_rate();
 271
 272         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 273
 274         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 275
 276         DCPTime t = time;
 277         while (t < end) {
 278
 279                 if (_audio_reel == _reels.end ()) {
 280                         /* This audio is off the end of the last reel; ignore it */
 281                         return;
 282                 }
 283
 284                 if (end <= _audio_reel->period().to) {
 285                         /* Easy case: we can write all the audio to this reel */
 286                         _audio_reel->write (audio);
 287                         t = end;
 288                 } else if (_audio_reel->period().to <= t) {
 289                         /* This reel is entirely before the start of our audio; just skip the reel */
 290                         ++_audio_reel;
 291                 } else {
 292                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 293                         DCPTime part_lengths[2] = {
 294                                 _audio_reel->period().to - t,
 295                                 end - _audio_reel->period().to
 296                         };
 297
 298                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 299                         Frame part_frames[2] = {
 300                                 part_lengths[0].frames_ceil(afr),
 301                                 part_lengths[1].frames_floor(afr)
 302                         };
 303
 304                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 305
 306                         if (part_frames[0]) {
 307                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 308                                 _audio_reel->write (part);
 309                         }
 310
 311                         if (part_frames[1]) {
 312                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 313                         } else {
 314                                 audio.reset ();
 315                         }
 316
 317                         ++_audio_reel;
 318                         t += part_lengths[0];
 319                 }
 320         }
 321 }
 322
 323
 324 void
 325 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 326 {
 327         if (_atmos_reel->period().to == time) {
 328                 ++_atmos_reel;
 329                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 330         }
 331
 332         /* We assume that we get a video frame's worth of data here */
 333         _atmos_reel->write (atmos, metadata);
 334 }
 335
 336
 337 /** Caller must hold a lock on _state_mutex */
 338 bool
 339 Writer::have_sequenced_image_at_queue_head ()
 340 {
 341         if (_queue.empty ()) {
 342                 return false;
 343         }
 344
 345         _queue.sort ();
 346         auto const & f = _queue.front();
 347         return _last_written[f.reel].next(f);
 348 }
 349
 350
 351 bool
 352 Writer::LastWritten::next (QueueItem qi) const
 353 {
 354         if (qi.eyes == Eyes::BOTH) {
 355                 /* 2D */
 356                 return qi.frame == (_frame + 1);
 357         }
 358
 359         /* 3D */
 360
 361         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 362                 return true;
 363         }
 364
 365         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 366                 return true;
 367         }
 368
 369         return false;
 370 }
 371
 372
 373 void
 374 Writer::LastWritten::update (QueueItem qi)
 375 {
 376         _frame = qi.frame;
 377         _eyes = qi.eyes;
 378 }
 379
 380
 381 void
 382 Writer::thread ()
 383 try
 384 {
 385         start_of_thread ("Writer");
 386
 387         while (true)
 388         {
 389                 boost::mutex::scoped_lock lock (_state_mutex);
 390
 391                 while (true) {
 392
 393                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 394                                 /* We've got something to do: go and do it */
 395                                 break;
 396                         }
 397
 398                         /* Nothing to do: wait until something happens which may indicate that we do */
 399                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 400                         _empty_condition.wait (lock);
 401                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 402                 }
 403
 404                 /* We stop here if we have been asked to finish, and if either the queue
 405                    is empty or we do not have a sequenced image at its head (if this is the
 406                    case we will never terminate as no new frames will be sent once
 407                    _finish is true).
 408                 */
 409                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 410                         /* (Hopefully temporarily) log anything that was not written */
 411                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 412                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 413                                 for (auto const& i: _queue) {
 414                                         if (i.type == QueueItem::Type::FULL) {
 415                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 416                                         } else {
 417                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 418                                         }
 419                                 }
 420                         }
 421                         return;
 422                 }
 423
 424                 /* Write any frames that we can write; i.e. those that are in sequence. */
 425                 while (have_sequenced_image_at_queue_head ()) {
 426                         auto qi = _queue.front ();
 427                         _last_written[qi.reel].update (qi);
 428                         _queue.pop_front ();
 429                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 430                                 --_queued_full_in_memory;
 431                         }
 432
 433                         lock.unlock ();
 434
 435                         auto& reel = _reels[qi.reel];
 436
 437                         switch (qi.type) {
 438                         case QueueItem::Type::FULL:
 439                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 440                                 if (!qi.encoded) {
 441                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 442                                 }
 443                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 444                                 ++_full_written;
 445                                 break;
 446                         case QueueItem::Type::FAKE:
 447                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 448                                 reel.fake_write (qi.size);
 449                                 ++_fake_written;
 450                                 break;
 451                         case QueueItem::Type::REPEAT:
 452                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 453                                 reel.repeat_write (qi.frame, qi.eyes);
 454                                 ++_repeat_written;
 455                                 break;
 456                         }
 457
 458                         lock.lock ();
 459                         _full_condition.notify_all ();
 460                 }
 461
 462                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 463                         /* Too many frames in memory which can't yet be written to the stream.
 464                            Write some FULL frames to disk.
 465                         */
 466
 467                         /* Find one from the back of the queue */
 468                         _queue.sort ();
 469                         auto i = _queue.rbegin ();
 470                         while (i != _queue.rend() && (i->type != QueueItem::Type::FULL || !i->encoded)) {
 471                                 ++i;
 472                         }
 473
 474                         DCPOMATIC_ASSERT (i != _queue.rend());
 475                         ++_pushed_to_disk;
 476                         /* For the log message below */
 477                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 478                         lock.unlock ();
 479
 480                         /* i is valid here, even though we don't hold a lock on the mutex,
 481                            since list iterators are unaffected by insertion and only this
 482                            thread could erase the last item in the list.
 483                         */
 484
 485                         LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
 486
 487                         i->encoded->write_via_temp (
 488                                 film()->j2c_path(i->reel, i->frame, i->eyes, true),
 489                                 film()->j2c_path(i->reel, i->frame, i->eyes, false)
 490                                 );
 491
 492                         lock.lock ();
 493                         i->encoded.reset ();
 494                         --_queued_full_in_memory;
 495                         _full_condition.notify_all ();
 496                 }
 497         }
 498 }
 499 catch (...)
 500 {
 501         store_current ();
 502 }
 503
 504
 505 void
 506 Writer::terminate_thread (bool can_throw)
 507 {
 508         boost::this_thread::disable_interruption dis;
 509
 510         boost::mutex::scoped_lock lock (_state_mutex);
 511
 512         _finish = true;
 513         _empty_condition.notify_all ();
 514         _full_condition.notify_all ();
 515         lock.unlock ();
 516
 517         try {
 518                 _thread.join ();
 519         } catch (...) {}
 520
 521         if (can_throw) {
 522                 rethrow ();
 523         }
 524 }
 525
 526
 527 void
 528 Writer::calculate_digests ()
 529 {
 530         auto job = _job.lock ();
 531         if (job) {
 532                 job->sub (_("Computing digests"));
 533         }
 534
 535         boost::asio::io_service service;
 536         boost::thread_group pool;
 537
 538         auto work = make_shared<boost::asio::io_service::work>(service);
 539
 540         int const threads = max (1, Config::instance()->master_encoding_threads());
 541
 542         for (int i = 0; i < threads; ++i) {
 543                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 544         }
 545
 546         std::function<void (float)> set_progress;
 547         if (job) {
 548                 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
 549         } else {
 550                 set_progress = [](float) {
 551                         boost::this_thread::interruption_point();
 552                 };
 553         }
 554
 555         for (auto& i: _reels) {
 556                 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
 557         }
 558         service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
 559
 560         work.reset ();
 561
 562         try {
 563                 pool.join_all ();
 564         } catch (boost::thread_interrupted) {
 565                 /* join_all was interrupted, so we need to interrupt the threads
 566                  * in our pool then try again to join them.
 567                  */
 568                 pool.interrupt_all ();
 569                 pool.join_all ();
 570         }
 571
 572         service.stop ();
 573 }
 574
 575
 576 /** @param output_dcp Path to DCP folder to write */
 577 void
 578 Writer::finish (boost::filesystem::path output_dcp)
 579 {
 580         if (_thread.joinable()) {
 581                 LOG_GENERAL_NC ("Terminating writer thread");
 582                 terminate_thread (true);
 583         }
 584
 585         LOG_GENERAL_NC ("Finishing ReelWriters");
 586
 587         for (auto& i: _reels) {
 588                 write_hanging_text (i);
 589                 i.finish (output_dcp);
 590         }
 591
 592         LOG_GENERAL_NC ("Writing XML");
 593
 594         dcp::DCP dcp (output_dcp);
 595
 596         auto cpl = make_shared<dcp::CPL>(
 597                 film()->dcp_name(),
 598                 film()->dcp_content_type()->libdcp_kind(),
 599                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 600                 );
 601
 602         dcp.add (cpl);
 603
 604         calculate_digests ();
 605
 606         /* Add reels */
 607
 608         for (auto& i: _reels) {
 609                 cpl->add (i.create_reel(_reel_assets, _fonts, _chosen_interop_font, output_dcp, _have_subtitles, _have_closed_captions));
 610         }
 611
 612         /* Add metadata */
 613
 614         auto creator = Config::instance()->dcp_creator();
 615         if (creator.empty()) {
 616                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 617         }
 618
 619         auto issuer = Config::instance()->dcp_issuer();
 620         if (issuer.empty()) {
 621                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 622         }
 623
 624         cpl->set_creator (creator);
 625         cpl->set_issuer (issuer);
 626
 627         cpl->set_ratings (film()->ratings());
 628
 629         vector<dcp::ContentVersion> cv;
 630         for (auto i: film()->content_versions()) {
 631                 cv.push_back (dcp::ContentVersion(i));
 632         }
 633         if (cv.empty()) {
 634                 cv = { dcp::ContentVersion("1") };
 635         }
 636         cpl->set_content_versions (cv);
 637
 638         cpl->set_full_content_title_text (film()->name());
 639         cpl->set_full_content_title_text_language (film()->name_language());
 640         if (film()->release_territory()) {
 641                 cpl->set_release_territory (*film()->release_territory());
 642         }
 643         cpl->set_version_number (film()->version_number());
 644         cpl->set_status (film()->status());
 645         if (film()->chain()) {
 646                 cpl->set_chain (*film()->chain());
 647         }
 648         if (film()->distributor()) {
 649                 cpl->set_distributor (*film()->distributor());
 650         }
 651         if (film()->facility()) {
 652                 cpl->set_facility (*film()->facility());
 653         }
 654         if (film()->luminance()) {
 655                 cpl->set_luminance (*film()->luminance());
 656         }
 657         if (film()->sign_language_video_language()) {
 658                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 659         }
 660
 661         auto ac = film()->mapped_audio_channels();
 662         dcp::MCASoundField field = (
 663                 find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSL)) != ac.end() ||
 664                 find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSR)) != ac.end()
 665                 ) ? dcp::MCASoundField::SEVEN_POINT_ONE : dcp::MCASoundField::FIVE_POINT_ONE;
 666
 667         dcp::MainSoundConfiguration msc (field, film()->audio_channels());
 668         for (auto i: ac) {
 669                 if (static_cast<int>(i) < film()->audio_channels()) {
 670                         msc.set_mapping (i, static_cast<dcp::Channel>(i));
 671                 }
 672         }
 673
 674         cpl->set_main_sound_configuration (msc.to_string());
 675         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 676         cpl->set_main_picture_stored_area (film()->frame_size());
 677
 678         auto active_area = film()->active_area();
 679         if (active_area.width > 0 && active_area.height > 0) {
 680                 /* It's not allowed to have a zero active area width or height */
 681                 cpl->set_main_picture_active_area (active_area);
 682         }
 683
 684         auto sl = film()->subtitle_languages().second;
 685         if (!sl.empty()) {
 686                 cpl->set_additional_subtitle_languages(sl);
 687         }
 688
 689         auto signer = Config::instance()->signer_chain();
 690         /* We did check earlier, but check again here to be on the safe side */
 691         string reason;
 692         if (!signer->valid (&reason)) {
 693                 throw InvalidSignerError (reason);
 694         }
 695
 696         dcp.set_issuer(issuer);
 697         dcp.set_creator(creator);
 698         dcp.set_annotation_text(film()->dcp_name());
 699
 700         dcp.write_xml (signer, Config::instance()->dcp_metadata_filename_format());
 701
 702         LOG_GENERAL (
 703                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 704                 );
 705
 706         write_cover_sheet (output_dcp);
 707 }
 708
 709
 710 void
 711 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 712 {
 713         auto const cover = film()->file("COVER_SHEET.txt");
 714         dcp::File f(cover, "w");
 715         if (!f) {
 716                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 717         }
 718
 719         auto text = Config::instance()->cover_sheet ();
 720         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 721         auto cpls = film()->cpls();
 722         if (!cpls.empty()) {
 723                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 724         }
 725         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 726         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 727
 728         auto audio_language = film()->audio_language();
 729         if (audio_language) {
 730                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 731         } else {
 732                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 733         }
 734
 735         auto subtitle_languages = film()->subtitle_languages();
 736         if (subtitle_languages.first) {
 737                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 738         } else {
 739                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 740         }
 741
 742         boost::uintmax_t size = 0;
 743         for (
 744                 auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
 745                 i != boost::filesystem::recursive_directory_iterator();
 746                 ++i) {
 747                 if (boost::filesystem::is_regular_file (i->path())) {
 748                         size += boost::filesystem::file_size (i->path());
 749                 }
 750         }
 751
 752         if (size > (1000000000L)) {
 753                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 754         } else {
 755                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 756         }
 757
 758         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 759         auto description = String::compose("%1.%2", ch.first, ch.second);
 760
 761         if (description == "0.0") {
 762                 description = _("None");
 763         } else if (description == "1.0") {
 764                 description = _("Mono");
 765         } else if (description == "2.0") {
 766                 description = _("Stereo");
 767         }
 768         boost::algorithm::replace_all (text, "$AUDIO", description);
 769
 770         auto const hmsf = film()->length().split(film()->video_frame_rate());
 771         string length;
 772         if (hmsf.h == 0 && hmsf.m == 0) {
 773                 length = String::compose("%1s", hmsf.s);
 774         } else if (hmsf.h == 0 && hmsf.m > 0) {
 775                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 776         } else if (hmsf.h > 0 && hmsf.m > 0) {
 777                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 778         }
 779
 780         boost::algorithm::replace_all (text, "$LENGTH", length);
 781
 782         f.checked_write(text.c_str(), text.length());
 783 }
 784
 785
 786 /** @param frame Frame index within the whole DCP.
 787  *  @return true if we can fake-write this frame.
 788  */
 789 bool
 790 Writer::can_fake_write (Frame frame) const
 791 {
 792         if (film()->encrypted()) {
 793                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 794                 return false;
 795         }
 796
 797         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 798            parameters in the asset writer.
 799         */
 800
 801         auto const & reel = _reels[video_reel(frame)];
 802
 803         /* Make frame relative to the start of the reel */
 804         frame -= reel.start ();
 805         return (frame != 0 && frame < reel.first_nonexistant_frame());
 806 }
 807
 808
 809 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 810 void
 811 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 812 {
 813         vector<ReelWriter>::iterator* reel = nullptr;
 814
 815         switch (type) {
 816         case TextType::OPEN_SUBTITLE:
 817                 reel = &_subtitle_reel;
 818                 _have_subtitles = true;
 819                 break;
 820         case TextType::CLOSED_CAPTION:
 821                 DCPOMATIC_ASSERT (track);
 822                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 823                 reel = &_caption_reels[*track];
 824                 _have_closed_captions.insert (*track);
 825                 break;
 826         default:
 827                 DCPOMATIC_ASSERT (false);
 828         }
 829
 830         DCPOMATIC_ASSERT (*reel != _reels.end());
 831         while ((*reel)->period().to <= period.from) {
 832                 ++(*reel);
 833                 DCPOMATIC_ASSERT (*reel != _reels.end());
 834                 write_hanging_text (**reel);
 835         }
 836
 837         auto back_off = [this](DCPTimePeriod period) {
 838                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 839                 return period;
 840         };
 841
 842         if (period.to > (*reel)->period().to) {
 843                 /* This text goes off the end of the reel.  Store parts of it that should go into
 844                  * other reels.
 845                  */
 846                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 847                         auto overlap = i->period().overlap(period);
 848                         if (overlap) {
 849                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 850                         }
 851                 }
 852                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 853                  * for subtitles being too close together.
 854                  */
 855                 period.to = (*reel)->period().to;
 856                 period = back_off(period);
 857         }
 858
 859         (*reel)->write(text, type, track, period, _fonts);
 860 }
 861
 862
 863 void
 864 Writer::write (vector<shared_ptr<Font>> fonts)
 865 {
 866         if (fonts.empty()) {
 867                 return;
 868         }
 869
 870         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 871         auto fix_id = [](string id) {
 872                 return id.empty() ? "font" : id;
 873         };
 874
 875         if (film()->interop()) {
 876                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 877                  * even write them as they upset some validators.  Set up _fonts so that every
 878                  * font used by any subtitle will be written with the same ID.
 879                  */
 880                 for (size_t i = 0; i < fonts.size(); ++i) {
 881                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 882                 }
 883                 _chosen_interop_font = fonts[0];
 884         } else {
 885                 set<string> used_ids;
 886
 887                 /* Return the index of a _N at the end of a string, or string::npos */
 888                 auto underscore_number_position = [](string s) {
 889                         auto last_underscore = s.find_last_of("_");
 890                         if (last_underscore == string::npos) {
 891                                 return string::npos;
 892                         }
 893
 894                         for (auto i = last_underscore + 1; i < s.size(); ++i) {
 895                                 if (!isdigit(s[i])) {
 896                                         return string::npos;
 897                                 }
 898                         }
 899
 900                         return last_underscore;
 901                 };
 902
 903                 /* Write fonts to _fonts, changing any duplicate IDs so that they are unique */
 904                 for (auto font: fonts) {
 905                         auto id = fix_id(font->id());
 906                         if (used_ids.find(id) == used_ids.end()) {
 907                                 /* This ID is unique so we can just use it as-is */
 908                                 _fonts.put(font, id);
 909                                 used_ids.insert(id);
 910                         } else {
 911                                 auto end = underscore_number_position(id);
 912                                 if (end == string::npos) {
 913                                         /* This string has no _N suffix, so add one */
 914                                         id += "_0";
 915                                         end = underscore_number_position(id);
 916                                 }
 917
 918                                 ++end;
 919
 920                                 /* Increment the suffix until we find a unique one */
 921                                 auto number = dcp::raw_convert<int>(id.substr(end));
 922                                 while (used_ids.find(id) != used_ids.end()) {
 923                                         ++number;
 924                                         id = String::compose("%1_%2", id.substr(0, end - 1), number);
 925                                 }
 926                                 used_ids.insert(id);
 927                         }
 928                         _fonts.put(font, id);
 929                 }
 930
 931                 DCPOMATIC_ASSERT(_fonts.map().size() == used_ids.size());
 932         }
 933 }
 934
 935
 936 bool
 937 operator< (QueueItem const & a, QueueItem const & b)
 938 {
 939         if (a.reel != b.reel) {
 940                 return a.reel < b.reel;
 941         }
 942
 943         if (a.frame != b.frame) {
 944                 return a.frame < b.frame;
 945         }
 946
 947         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 948 }
 949
 950
 951 bool
 952 operator== (QueueItem const & a, QueueItem const & b)
 953 {
 954         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 955 }
 956
 957
 958 void
 959 Writer::set_encoder_threads (int threads)
 960 {
 961         boost::mutex::scoped_lock lm (_state_mutex);
 962         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 963         _maximum_queue_size = threads * 16;
 964 }
 965
 966
 967 void
 968 Writer::write (ReferencedReelAsset asset)
 969 {
 970         _reel_assets.push_back (asset);
 971 }
 972
 973
 974 size_t
 975 Writer::video_reel (int frame) const
 976 {
 977         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 978         size_t i = 0;
 979         while (i < _reels.size() && !_reels[i].period().contains (t)) {
 980                 ++i;
 981         }
 982
 983         DCPOMATIC_ASSERT (i < _reels.size ());
 984         return i;
 985 }
 986
 987
 988 void
 989 Writer::set_digest_progress (Job* job, float progress)
 990 {
 991         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 992
 993         _digest_progresses[boost::this_thread::get_id()] = progress;
 994         float min_progress = FLT_MAX;
 995         for (auto const& i: _digest_progresses) {
 996                 min_progress = min (min_progress, i.second);
 997         }
 998
 999         job->set_progress (min_progress);
1000
1001         Waker waker;
1002         waker.nudge ();
1003
1004         boost::this_thread::interruption_point();
1005 }
1006
1007
1008 /** Calculate hashes for any referenced MXF assets which do not already have one */
1009 void
1010 Writer::calculate_referenced_digests (std::function<void (float)> set_progress)
1011 try
1012 {
1013         for (auto const& i: _reel_assets) {
1014                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1015                 if (file && !file->hash()) {
1016                         file->asset_ref().asset()->hash (set_progress);
1017                         file->set_hash (file->asset_ref().asset()->hash());
1018                 }
1019         }
1020 } catch (boost::thread_interrupted) {
1021         /* set_progress contains an interruption_point, so any of these methods
1022          * may throw thread_interrupted, at which point we just give up.
1023          */
1024 }
1025
1026
1027 void
1028 Writer::write_hanging_text (ReelWriter& reel)
1029 {
1030         vector<HangingText> new_hanging_texts;
1031         for (auto i: _hanging_texts) {
1032                 if (i.period.from == reel.period().from) {
1033                         reel.write (i.text, i.type, i.track, i.period, _fonts);
1034                 } else {
1035                         new_hanging_texts.push_back (i);
1036                 }
1037         }
1038         _hanging_texts = new_hanging_texts;
1039 }