src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <fstream>
  21 #include <cerrno>
  22 #include <libdcp/mono_picture_asset.h>
  23 #include <libdcp/stereo_picture_asset.h>
  24 #include <libdcp/sound_asset.h>
  25 #include <libdcp/reel.h>
  26 #include <libdcp/dcp.h>
  27 #include <libdcp/cpl.h>
  28 #include "writer.h"
  29 #include "compose.hpp"
  30 #include "film.h"
  31 #include "ratio.h"
  32 #include "log.h"
  33 #include "dcp_video_frame.h"
  34 #include "dcp_content_type.h"
  35 #include "player.h"
  36 #include "audio_mapping.h"
  37 #include "config.h"
  38 #include "job.h"
  39 #include "cross.h"
  40
  41 #include "i18n.h"
  42
  43 using std::make_pair;
  44 using std::pair;
  45 using std::string;
  46 using std::list;
  47 using std::cout;
  48 using boost::shared_ptr;
  49 using boost::weak_ptr;
  50
  51 int const Writer::_maximum_frames_in_memory = Config::instance()->num_local_encoding_threads() + 4;
  52
  53 Writer::Writer (shared_ptr<const Film> f, weak_ptr<Job> j)
  54         : _film (f)
  55         , _job (j)
  56         , _first_nonexistant_frame (0)
  57         , _thread (0)
  58         , _finish (false)
  59         , _queued_full_in_memory (0)
  60         , _last_written_frame (-1)
  61         , _last_written_eyes (EYES_RIGHT)
  62         , _full_written (0)
  63         , _fake_written (0)
  64         , _repeat_written (0)
  65         , _pushed_to_disk (0)
  66 {
  67         /* Remove any old DCP */
  68         boost::filesystem::remove_all (_film->dir (_film->dcp_name ()));
  69
  70         shared_ptr<Job> job = _job.lock ();
  71         assert (job);
  72
  73         job->sub (_("Checking existing image data"));
  74         check_existing_picture_mxf ();
  75
  76         /* Create our picture asset in a subdirectory, named according to those
  77            film's parameters which affect the video output.  We will hard-link
  78            it into the DCP later.
  79         */
  80
  81         if (_film->three_d ()) {
  82                 _picture_asset.reset (new libdcp::StereoPictureAsset (_film->internal_video_mxf_dir (), _film->internal_video_mxf_filename ()));
  83         } else {
  84                 _picture_asset.reset (new libdcp::MonoPictureAsset (_film->internal_video_mxf_dir (), _film->internal_video_mxf_filename ()));
  85         }
  86
  87         _picture_asset->set_edit_rate (_film->video_frame_rate ());
  88         _picture_asset->set_size (fit_ratio_within (_film->container()->ratio(), _film->full_frame ()));
  89
  90         if (_film->encrypted ()) {
  91                 _picture_asset->set_key (_film->key ());
  92         }
  93
  94         _picture_asset_writer = _picture_asset->start_write (_first_nonexistant_frame > 0);
  95
  96         /* Write the sound asset into the film directory so that we leave the creation
  97            of the DCP directory until the last minute.  Some versions of windows inexplicably
  98            don't like overwriting existing files here, so try to remove it using boost.
  99         */
 100         boost::system::error_code ec;
 101         boost::filesystem::remove_all (_film->file (_film->audio_mxf_filename ()), ec);
 102         if (ec) {
 103                 _film->log()->log (
 104                         String::compose (
 105                                 "Could not remove existing audio MXF file %1 (%2)",
 106                                 _film->file (_film->audio_mxf_filename ()),
 107                                 ec.value ())
 108                         );
 109         }
 110
 111         _sound_asset.reset (new libdcp::SoundAsset (_film->directory (), _film->audio_mxf_filename ()));
 112         _sound_asset->set_edit_rate (_film->video_frame_rate ());
 113         _sound_asset->set_channels (_film->audio_channels ());
 114         _sound_asset->set_sampling_rate (_film->audio_frame_rate ());
 115
 116         if (_film->encrypted ()) {
 117                 _sound_asset->set_key (_film->key ());
 118         }
 119
 120         _sound_asset_writer = _sound_asset->start_write ();
 121
 122         _thread = new boost::thread (boost::bind (&Writer::thread, this));
 123
 124         job->sub (_("Encoding image data"));
 125 }
 126
 127 void
 128 Writer::write (shared_ptr<const EncodedData> encoded, int frame, Eyes eyes)
 129 {
 130         boost::mutex::scoped_lock lock (_mutex);
 131
 132         QueueItem qi;
 133         qi.type = QueueItem::FULL;
 134         qi.encoded = encoded;
 135         qi.frame = frame;
 136
 137         if (_film->three_d() && eyes == EYES_BOTH) {
 138                 /* 2D material in a 3D DCP; fake the 3D */
 139                 qi.eyes = EYES_LEFT;
 140                 _queue.push_back (qi);
 141                 ++_queued_full_in_memory;
 142                 qi.eyes = EYES_RIGHT;
 143                 _queue.push_back (qi);
 144                 ++_queued_full_in_memory;
 145         } else {
 146                 qi.eyes = eyes;
 147                 _queue.push_back (qi);
 148                 ++_queued_full_in_memory;
 149         }
 150
 151         _condition.notify_all ();
 152 }
 153
 154 void
 155 Writer::fake_write (int frame, Eyes eyes)
 156 {
 157         boost::mutex::scoped_lock lock (_mutex);
 158
 159         FILE* ifi = fopen_boost (_film->info_path (frame, eyes), "r");
 160         libdcp::FrameInfo info (ifi);
 161         fclose (ifi);
 162
 163         QueueItem qi;
 164         qi.type = QueueItem::FAKE;
 165         qi.size = info.size;
 166         qi.frame = frame;
 167         if (_film->three_d() && eyes == EYES_BOTH) {
 168                 qi.eyes = EYES_LEFT;
 169                 _queue.push_back (qi);
 170                 qi.eyes = EYES_RIGHT;
 171                 _queue.push_back (qi);
 172         } else {
 173                 qi.eyes = eyes;
 174                 _queue.push_back (qi);
 175         }
 176
 177         _condition.notify_all ();
 178 }
 179
 180 /** This method is not thread safe */
 181 void
 182 Writer::write (shared_ptr<const AudioBuffers> audio)
 183 {
 184         _sound_asset_writer->write (audio->data(), audio->frames());
 185 }
 186
 187 /** This must be called from Writer::thread() with an appropriate lock held */
 188 bool
 189 Writer::have_sequenced_image_at_queue_head ()
 190 {
 191         if (_queue.empty ()) {
 192                 return false;
 193         }
 194
 195         _queue.sort ();
 196
 197         /* The queue should contain only EYES_LEFT/EYES_RIGHT pairs or EYES_BOTH */
 198
 199         if (_queue.front().eyes == EYES_BOTH) {
 200                 /* 2D */
 201                 return _queue.front().frame == (_last_written_frame + 1);
 202         }
 203
 204         /* 3D */
 205
 206         if (_last_written_eyes == EYES_LEFT && _queue.front().frame == _last_written_frame && _queue.front().eyes == EYES_RIGHT) {
 207                 return true;
 208         }
 209
 210         if (_last_written_eyes == EYES_RIGHT && _queue.front().frame == (_last_written_frame + 1) && _queue.front().eyes == EYES_LEFT) {
 211                 return true;
 212         }
 213
 214         return false;
 215 }
 216
 217 void
 218 Writer::thread ()
 219 try
 220 {
 221         while (1)
 222         {
 223                 boost::mutex::scoped_lock lock (_mutex);
 224
 225                 while (1) {
 226
 227                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 228                                 break;
 229                         }
 230
 231                         TIMING (N_("writer sleeps with a queue of %1"), _queue.size());
 232                         _condition.wait (lock);
 233                         TIMING (N_("writer wakes with a queue of %1"), _queue.size());
 234                 }
 235
 236                 if (_finish && _queue.empty()) {
 237                         return;
 238                 }
 239
 240                 /* Write any frames that we can write; i.e. those that are in sequence. */
 241                 while (have_sequenced_image_at_queue_head ()) {
 242                         QueueItem qi = _queue.front ();
 243                         _queue.pop_front ();
 244                         if (qi.type == QueueItem::FULL && qi.encoded) {
 245                                 --_queued_full_in_memory;
 246                         }
 247
 248                         lock.unlock ();
 249                         switch (qi.type) {
 250                         case QueueItem::FULL:
 251                         {
 252                                 _film->log()->log (String::compose (N_("Writer FULL-writes %1 to MXF"), qi.frame));
 253                                 if (!qi.encoded) {
 254                                         qi.encoded.reset (new EncodedData (_film->j2c_path (qi.frame, qi.eyes, false)));
 255                                 }
 256
 257                                 libdcp::FrameInfo fin = _picture_asset_writer->write (qi.encoded->data(), qi.encoded->size());
 258                                 qi.encoded->write_info (_film, qi.frame, qi.eyes, fin);
 259                                 _last_written[qi.eyes] = qi.encoded;
 260                                 ++_full_written;
 261                                 break;
 262                         }
 263                         case QueueItem::FAKE:
 264                                 _film->log()->log (String::compose (N_("Writer FAKE-writes %1 to MXF"), qi.frame));
 265                                 _picture_asset_writer->fake_write (qi.size);
 266                                 _last_written[qi.eyes].reset ();
 267                                 ++_fake_written;
 268                                 break;
 269                         case QueueItem::REPEAT:
 270                         {
 271                                 _film->log()->log (String::compose (N_("Writer REPEAT-writes %1 to MXF"), qi.frame));
 272                                 libdcp::FrameInfo fin = _picture_asset_writer->write (
 273                                         _last_written[qi.eyes]->data(),
 274                                         _last_written[qi.eyes]->size()
 275                                         );
 276
 277                                 _last_written[qi.eyes]->write_info (_film, qi.frame, qi.eyes, fin);
 278                                 ++_repeat_written;
 279                                 break;
 280                         }
 281                         }
 282                         lock.lock ();
 283
 284                         _last_written_frame = qi.frame;
 285                         _last_written_eyes = qi.eyes;
 286
 287                         if (_film->length()) {
 288                                 shared_ptr<Job> job = _job.lock ();
 289                                 assert (job);
 290                                 int total = _film->time_to_video_frames (_film->length ());
 291                                 if (_film->three_d ()) {
 292                                         /* _full_written and so on are incremented for each eye, so we need to double the total
 293                                            frames to get the correct progress.
 294                                         */
 295                                         total *= 2;
 296                                 }
 297                                 job->set_progress (float (_full_written + _fake_written + _repeat_written) / total);
 298                         }
 299                 }
 300
 301                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 302                         /* Too many frames in memory which can't yet be written to the stream.
 303                            Write some FULL frames to disk.
 304                         */
 305
 306                         /* Find one from the back of the queue */
 307                         _queue.sort ();
 308                         list<QueueItem>::reverse_iterator i = _queue.rbegin ();
 309                         while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
 310                                 ++i;
 311                         }
 312
 313                         assert (i != _queue.rend());
 314                         QueueItem qi = *i;
 315
 316                         ++_pushed_to_disk;
 317
 318                         lock.unlock ();
 319
 320                         _film->log()->log (
 321                                 String::compose (
 322                                         "Writer full (awaiting %1 [last eye was %2]); pushes %3 to disk",
 323                                         _last_written_frame + 1,
 324                                         _last_written_eyes, qi.frame)
 325                                 );
 326
 327                         qi.encoded->write (_film, qi.frame, qi.eyes);
 328                         lock.lock ();
 329                         qi.encoded.reset ();
 330                         --_queued_full_in_memory;
 331                 }
 332         }
 333 }
 334 catch (...)
 335 {
 336         store_current ();
 337 }
 338
 339 void
 340 Writer::finish ()
 341 {
 342         if (!_thread) {
 343                 return;
 344         }
 345
 346         boost::mutex::scoped_lock lock (_mutex);
 347         _finish = true;
 348         _condition.notify_all ();
 349         lock.unlock ();
 350
 351         _thread->join ();
 352         rethrow ();
 353
 354         delete _thread;
 355         _thread = 0;
 356
 357         _picture_asset_writer->finalize ();
 358         _sound_asset_writer->finalize ();
 359
 360         int const frames = _last_written_frame + 1;
 361
 362         _picture_asset->set_duration (frames);
 363
 364         /* Hard-link the video MXF into the DCP */
 365         boost::filesystem::path video_from;
 366         video_from /= _film->internal_video_mxf_dir();
 367         video_from /= _film->internal_video_mxf_filename();
 368
 369         boost::filesystem::path video_to;
 370         video_to /= _film->dir (_film->dcp_name());
 371         video_to /= _film->video_mxf_filename ();
 372
 373         boost::system::error_code ec;
 374         boost::filesystem::create_hard_link (video_from, video_to, ec);
 375         if (ec) {
 376                 /* hard link failed; copy instead */
 377                 boost::filesystem::copy_file (video_from, video_to);
 378                 _film->log()->log ("Hard-link failed; fell back to copying");
 379         }
 380
 381         /* And update the asset */
 382
 383         _picture_asset->set_directory (_film->dir (_film->dcp_name ()));
 384         _picture_asset->set_file_name (_film->video_mxf_filename ());
 385
 386         /* Move the audio MXF into the DCP */
 387
 388         boost::filesystem::path audio_to;
 389         audio_to /= _film->dir (_film->dcp_name ());
 390         audio_to /= _film->audio_mxf_filename ();
 391
 392         boost::filesystem::rename (_film->file (_film->audio_mxf_filename ()), audio_to, ec);
 393         if (ec) {
 394                 throw FileError (
 395                         String::compose (_("could not move audio MXF into the DCP (%1)"), ec.value ()), _film->file (_film->audio_mxf_filename ())
 396                         );
 397         }
 398
 399         _sound_asset->set_directory (_film->dir (_film->dcp_name ()));
 400         _sound_asset->set_duration (frames);
 401
 402         libdcp::DCP dcp (_film->dir (_film->dcp_name()));
 403
 404         shared_ptr<libdcp::CPL> cpl (
 405                 new libdcp::CPL (
 406                         _film->dir (_film->dcp_name()),
 407                         _film->dcp_name(),
 408                         _film->dcp_content_type()->libdcp_kind (),
 409                         frames,
 410                         _film->video_frame_rate ()
 411                         )
 412                 );
 413
 414         dcp.add_cpl (cpl);
 415
 416         cpl->add_reel (shared_ptr<libdcp::Reel> (new libdcp::Reel (
 417                                                          _picture_asset,
 418                                                          _sound_asset,
 419                                                          shared_ptr<libdcp::SubtitleAsset> ()
 420                                                          )
 421                                ));
 422
 423         shared_ptr<Job> job = _job.lock ();
 424         assert (job);
 425
 426         job->sub (_("Computing image digest"));
 427         _picture_asset->compute_digest (boost::bind (&Job::set_progress, job.get(), _1, false));
 428
 429         job->sub (_("Computing audio digest"));
 430         _sound_asset->compute_digest (boost::bind (&Job::set_progress, job.get(), _1, false));
 431
 432         libdcp::XMLMetadata meta = Config::instance()->dcp_metadata ();
 433         meta.set_issue_date_now ();
 434         dcp.write_xml (_film->interop (), meta, _film->is_signed() ? make_signer () : shared_ptr<const libdcp::Signer> ());
 435
 436         _film->log()->log (String::compose (N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT; %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk));
 437 }
 438
 439 /** Tell the writer that frame `f' should be a repeat of the frame before it */
 440 void
 441 Writer::repeat (int f, Eyes e)
 442 {
 443         boost::mutex::scoped_lock lock (_mutex);
 444
 445         QueueItem qi;
 446         qi.type = QueueItem::REPEAT;
 447         qi.frame = f;
 448         if (_film->three_d() && e == EYES_BOTH) {
 449                 qi.eyes = EYES_LEFT;
 450                 _queue.push_back (qi);
 451                 qi.eyes = EYES_RIGHT;
 452                 _queue.push_back (qi);
 453         } else {
 454                 qi.eyes = e;
 455                 _queue.push_back (qi);
 456         }
 457
 458         _condition.notify_all ();
 459 }
 460
 461 bool
 462 Writer::check_existing_picture_mxf_frame (FILE* mxf, int f, Eyes eyes)
 463 {
 464         /* Read the frame info as written */
 465         FILE* ifi = fopen_boost (_film->info_path (f, eyes), "r");
 466         if (!ifi) {
 467                 _film->log()->log (String::compose ("Existing frame %1 has no info file", f));
 468                 return false;
 469         }
 470
 471         libdcp::FrameInfo info (ifi);
 472         fclose (ifi);
 473         if (info.size == 0) {
 474                 _film->log()->log (String::compose ("Existing frame %1 has no info file", f));
 475                 return false;
 476         }
 477
 478         /* Read the data from the MXF and hash it */
 479         dcpomatic_fseek (mxf, info.offset, SEEK_SET);
 480         EncodedData data (info.size);
 481         size_t const read = fread (data.data(), 1, data.size(), mxf);
 482         if (read != static_cast<size_t> (data.size ())) {
 483                 _film->log()->log (String::compose ("Existing frame %1 is incomplete", f));
 484                 return false;
 485         }
 486
 487         string const existing_hash = md5_digest (data.data(), data.size());
 488         if (existing_hash != info.hash) {
 489                 _film->log()->log (String::compose ("Existing frame %1 failed hash check", f));
 490                 return false;
 491         }
 492
 493         return true;
 494 }
 495
 496 void
 497 Writer::check_existing_picture_mxf ()
 498 {
 499         /* Try to open the existing MXF */
 500         boost::filesystem::path p;
 501         p /= _film->internal_video_mxf_dir ();
 502         p /= _film->internal_video_mxf_filename ();
 503         FILE* mxf = fopen_boost (p, "rb");
 504         if (!mxf) {
 505                 _film->log()->log (String::compose ("Could not open existing MXF at %1 (errno=%2)", p.string(), errno));
 506                 return;
 507         }
 508
 509         int N = 0;
 510         for (boost::filesystem::directory_iterator i (_film->info_dir ()); i != boost::filesystem::directory_iterator (); ++i) {
 511                 ++N;
 512         }
 513
 514         while (1) {
 515
 516                 shared_ptr<Job> job = _job.lock ();
 517                 assert (job);
 518
 519                 job->set_progress (float (_first_nonexistant_frame) / N);
 520
 521                 if (_film->three_d ()) {
 522                         if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_LEFT)) {
 523                                 break;
 524                         }
 525                         if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_RIGHT)) {
 526                                 break;
 527                         }
 528                 } else {
 529                         if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_BOTH)) {
 530                                 break;
 531                         }
 532                 }
 533
 534                 _film->log()->log (String::compose ("Have existing frame %1", _first_nonexistant_frame));
 535                 ++_first_nonexistant_frame;
 536         }
 537
 538         fclose (mxf);
 539 }
 540
 541 /** @param frame Frame index.
 542  *  @return true if we can fake-write this frame.
 543  */
 544 bool
 545 Writer::can_fake_write (int frame) const
 546 {
 547         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 548            parameters in the MXF writer.
 549         */
 550         return (frame != 0 && frame < _first_nonexistant_frame);
 551 }
 552
 553 bool
 554 operator< (QueueItem const & a, QueueItem const & b)
 555 {
 556         if (a.frame != b.frame) {
 557                 return a.frame < b.frame;
 558         }
 559
 560         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 561 }
 562
 563 bool
 564 operator== (QueueItem const & a, QueueItem const & b)
 565 {
 566         return a.frame == b.frame && a.eyes == b.eyes;
 567 }