src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <fstream>
  21 #include <cerrno>
  22 #include <dcp/mono_picture_mxf.h>
  23 #include <dcp/stereo_picture_mxf.h>
  24 #include <dcp/sound_mxf.h>
  25 #include <dcp/sound_mxf_writer.h>
  26 #include <dcp/reel.h>
  27 #include <dcp/reel_mono_picture_asset.h>
  28 #include <dcp/reel_stereo_picture_asset.h>
  29 #include <dcp/reel_sound_asset.h>
  30 #include <dcp/dcp.h>
  31 #include <dcp/cpl.h>
  32 #include "writer.h"
  33 #include "compose.hpp"
  34 #include "film.h"
  35 #include "ratio.h"
  36 #include "log.h"
  37 #include "dcp_video.h"
  38 #include "dcp_content_type.h"
  39 #include "audio_mapping.h"
  40 #include "config.h"
  41 #include "job.h"
  42 #include "cross.h"
  43 #include "audio_buffers.h"
  44 #include "md5_digester.h"
  45 #include "encoded_data.h"
  46
  47 #include "i18n.h"
  48
  49 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
  50 #define LOG_TIMING(...) _film->log()->microsecond_log (String::compose (__VA_ARGS__), Log::TYPE_TIMING);
  51 #define LOG_WARNING_NC(...) _film->log()->log (__VA_ARGS__, Log::TYPE_WARNING);
  52
  53 /* OS X strikes again */
  54 #undef set_key
  55
  56 using std::make_pair;
  57 using std::pair;
  58 using std::string;
  59 using std::list;
  60 using std::cout;
  61 using std::stringstream;
  62 using boost::shared_ptr;
  63 using boost::weak_ptr;
  64 using boost::dynamic_pointer_cast;
  65
  66 int const Writer::_maximum_frames_in_memory = Config::instance()->num_local_encoding_threads() + 4;
  67
  68 Writer::Writer (shared_ptr<const Film> f, weak_ptr<Job> j)
  69         : _film (f)
  70         , _job (j)
  71         , _first_nonexistant_frame (0)
  72         , _thread (0)
  73         , _finish (false)
  74         , _queued_full_in_memory (0)
  75         , _last_written_frame (-1)
  76         , _last_written_eyes (EYES_RIGHT)
  77         , _full_written (0)
  78         , _fake_written (0)
  79         , _pushed_to_disk (0)
  80 {
  81         /* Remove any old DCP */
  82         boost::filesystem::remove_all (_film->dir (_film->dcp_name ()));
  83
  84         shared_ptr<Job> job = _job.lock ();
  85         assert (job);
  86
  87         job->sub (_("Checking existing image data"));
  88         check_existing_picture_mxf ();
  89
  90         /* Create our picture asset in a subdirectory, named according to those
  91            film's parameters which affect the video output.  We will hard-link
  92            it into the DCP later.
  93         */
  94
  95         if (_film->three_d ()) {
  96                 _picture_mxf.reset (new dcp::StereoPictureMXF (dcp::Fraction (_film->video_frame_rate (), 1)));
  97         } else {
  98                 _picture_mxf.reset (new dcp::MonoPictureMXF (dcp::Fraction (_film->video_frame_rate (), 1)));
  99         }
 100
 101         _picture_mxf->set_size (_film->frame_size ());
 102
 103         if (_film->encrypted ()) {
 104                 _picture_mxf->set_key (_film->key ());
 105         }
 106
 107         _picture_mxf_writer = _picture_mxf->start_write (
 108                 _film->internal_video_mxf_dir() / _film->internal_video_mxf_filename(),
 109                 _film->interop() ? dcp::INTEROP : dcp::SMPTE,
 110                 _first_nonexistant_frame > 0
 111                 );
 112
 113         if (_film->audio_channels ()) {
 114                 _sound_mxf.reset (new dcp::SoundMXF (dcp::Fraction (_film->video_frame_rate(), 1), _film->audio_frame_rate (), _film->audio_channels ()));
 115
 116                 if (_film->encrypted ()) {
 117                         _sound_mxf->set_key (_film->key ());
 118                 }
 119
 120                 /* Write the sound MXF into the film directory so that we leave the creation
 121                    of the DCP directory until the last minute.
 122                 */
 123                 _sound_mxf_writer = _sound_mxf->start_write (_film->directory() / _film->audio_mxf_filename(), _film->interop() ? dcp::INTEROP : dcp::SMPTE);
 124         }
 125
 126         _thread = new boost::thread (boost::bind (&Writer::thread, this));
 127
 128         job->sub (_("Encoding image data"));
 129 }
 130
 131 Writer::~Writer ()
 132 {
 133         terminate_thread (false);
 134 }
 135
 136 void
 137 Writer::write (shared_ptr<const EncodedData> encoded, int frame, Eyes eyes)
 138 {
 139         boost::mutex::scoped_lock lock (_mutex);
 140
 141         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 142                 /* The queue is too big; wait until that is sorted out */
 143                 _full_condition.wait (lock);
 144         }
 145
 146         QueueItem qi;
 147         qi.type = QueueItem::FULL;
 148         qi.encoded = encoded;
 149         qi.frame = frame;
 150
 151         if (_film->three_d() && eyes == EYES_BOTH) {
 152                 /* 2D material in a 3D DCP; fake the 3D */
 153                 qi.eyes = EYES_LEFT;
 154                 _queue.push_back (qi);
 155                 ++_queued_full_in_memory;
 156                 qi.eyes = EYES_RIGHT;
 157                 _queue.push_back (qi);
 158                 ++_queued_full_in_memory;
 159         } else {
 160                 qi.eyes = eyes;
 161                 _queue.push_back (qi);
 162                 ++_queued_full_in_memory;
 163         }
 164
 165         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 166         _empty_condition.notify_all ();
 167 }
 168
 169 void
 170 Writer::fake_write (int frame, Eyes eyes)
 171 {
 172         boost::mutex::scoped_lock lock (_mutex);
 173
 174         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 175                 /* The queue is too big; wait until that is sorted out */
 176                 _full_condition.wait (lock);
 177         }
 178
 179         FILE* ifi = fopen_boost (_film->info_path (frame, eyes), "r");
 180         dcp::FrameInfo info (ifi);
 181         fclose (ifi);
 182
 183         QueueItem qi;
 184         qi.type = QueueItem::FAKE;
 185         qi.size = info.size;
 186         qi.frame = frame;
 187         if (_film->three_d() && eyes == EYES_BOTH) {
 188                 qi.eyes = EYES_LEFT;
 189                 _queue.push_back (qi);
 190                 qi.eyes = EYES_RIGHT;
 191                 _queue.push_back (qi);
 192         } else {
 193                 qi.eyes = eyes;
 194                 _queue.push_back (qi);
 195         }
 196
 197         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 198         _empty_condition.notify_all ();
 199 }
 200
 201 /** This method is not thread safe */
 202 void
 203 Writer::write (shared_ptr<const AudioBuffers> audio)
 204 {
 205         if (_sound_mxf_writer) {
 206                 _sound_mxf_writer->write (audio->data(), audio->frames());
 207         }
 208 }
 209
 210 /** This must be called from Writer::thread() with an appropriate lock held */
 211 bool
 212 Writer::have_sequenced_image_at_queue_head ()
 213 {
 214         if (_queue.empty ()) {
 215                 return false;
 216         }
 217
 218         _queue.sort ();
 219
 220         /* The queue should contain only EYES_LEFT/EYES_RIGHT pairs or EYES_BOTH */
 221
 222         if (_queue.front().eyes == EYES_BOTH) {
 223                 /* 2D */
 224                 return _queue.front().frame == (_last_written_frame + 1);
 225         }
 226
 227         /* 3D */
 228
 229         if (_last_written_eyes == EYES_LEFT && _queue.front().frame == _last_written_frame && _queue.front().eyes == EYES_RIGHT) {
 230                 return true;
 231         }
 232
 233         if (_last_written_eyes == EYES_RIGHT && _queue.front().frame == (_last_written_frame + 1) && _queue.front().eyes == EYES_LEFT) {
 234                 return true;
 235         }
 236
 237         return false;
 238 }
 239
 240 void
 241 Writer::thread ()
 242 try
 243 {
 244         while (true)
 245         {
 246                 boost::mutex::scoped_lock lock (_mutex);
 247
 248                 while (true) {
 249
 250                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 251                                 /* We've got something to do: go and do it */
 252                                 break;
 253                         }
 254
 255                         /* Nothing to do: wait until something happens which may indicate that we do */
 256                         LOG_TIMING (N_("writer sleeps with a queue of %1"), _queue.size());
 257                         _empty_condition.wait (lock);
 258                         LOG_TIMING (N_("writer wakes with a queue of %1"), _queue.size());
 259                 }
 260
 261                 if (_finish && _queue.empty()) {
 262                         return;
 263                 }
 264
 265                 /* Write any frames that we can write; i.e. those that are in sequence. */
 266                 while (have_sequenced_image_at_queue_head ()) {
 267                         QueueItem qi = _queue.front ();
 268                         _queue.pop_front ();
 269                         if (qi.type == QueueItem::FULL && qi.encoded) {
 270                                 --_queued_full_in_memory;
 271                         }
 272
 273                         lock.unlock ();
 274                         switch (qi.type) {
 275                         case QueueItem::FULL:
 276                         {
 277                                 LOG_GENERAL (N_("Writer FULL-writes %1 to MXF"), qi.frame);
 278                                 if (!qi.encoded) {
 279                                         qi.encoded.reset (new EncodedData (_film->j2c_path (qi.frame, qi.eyes, false)));
 280                                 }
 281
 282                                 dcp::FrameInfo fin = _picture_mxf_writer->write (qi.encoded->data(), qi.encoded->size());
 283                                 qi.encoded->write_info (_film, qi.frame, qi.eyes, fin);
 284                                 _last_written[qi.eyes] = qi.encoded;
 285                                 ++_full_written;
 286                                 break;
 287                         }
 288                         case QueueItem::FAKE:
 289                                 LOG_GENERAL (N_("Writer FAKE-writes %1 to MXF"), qi.frame);
 290                                 _picture_mxf_writer->fake_write (qi.size);
 291                                 _last_written[qi.eyes].reset ();
 292                                 ++_fake_written;
 293                                 break;
 294                         }
 295                         lock.lock ();
 296
 297                         _last_written_frame = qi.frame;
 298                         _last_written_eyes = qi.eyes;
 299
 300                         shared_ptr<Job> job = _job.lock ();
 301                         assert (job);
 302                         int64_t total = _film->length().frames (_film->video_frame_rate ());
 303                         if (_film->three_d ()) {
 304                                 /* _full_written and so on are incremented for each eye, so we need to double the total
 305                                    frames to get the correct progress.
 306                                 */
 307                                 total *= 2;
 308                         }
 309                         if (total) {
 310                                 job->set_progress (float (_full_written + _fake_written) / total);
 311                         }
 312                 }
 313
 314                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 315                         /* Too many frames in memory which can't yet be written to the stream.
 316                            Write some FULL frames to disk.
 317                         */
 318
 319                         /* Find one from the back of the queue */
 320                         _queue.sort ();
 321                         list<QueueItem>::reverse_iterator i = _queue.rbegin ();
 322                         while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
 323                                 ++i;
 324                         }
 325
 326                         assert (i != _queue.rend());
 327                         QueueItem qi = *i;
 328
 329                         ++_pushed_to_disk;
 330
 331                         lock.unlock ();
 332
 333                         LOG_GENERAL (
 334                                 "Writer full (awaiting %1 [last eye was %2]); pushes %3 to disk",
 335                                 _last_written_frame + 1,
 336                                 _last_written_eyes, qi.frame
 337                                 );
 338
 339                         qi.encoded->write (_film, qi.frame, qi.eyes);
 340                         lock.lock ();
 341                         qi.encoded.reset ();
 342                         --_queued_full_in_memory;
 343                 }
 344
 345                 /* The queue has probably just gone down a bit; notify anything wait()ing on _full_condition */
 346                 _full_condition.notify_all ();
 347         }
 348 }
 349 catch (...)
 350 {
 351         store_current ();
 352 }
 353
 354 void
 355 Writer::terminate_thread (bool can_throw)
 356 {
 357         boost::mutex::scoped_lock lock (_mutex);
 358         if (_thread == 0) {
 359                 return;
 360         }
 361
 362         _finish = true;
 363         _empty_condition.notify_all ();
 364         _full_condition.notify_all ();
 365         lock.unlock ();
 366
 367         _thread->join ();
 368         if (can_throw) {
 369                 rethrow ();
 370         }
 371
 372         delete _thread;
 373         _thread = 0;
 374 }
 375
 376 void
 377 Writer::finish ()
 378 {
 379         if (!_thread) {
 380                 return;
 381         }
 382
 383         terminate_thread (true);
 384
 385         _picture_mxf_writer->finalize ();
 386         if (_sound_mxf_writer) {
 387                 _sound_mxf_writer->finalize ();
 388         }
 389
 390         /* Hard-link the video MXF into the DCP */
 391         boost::filesystem::path video_from;
 392         video_from /= _film->internal_video_mxf_dir();
 393         video_from /= _film->internal_video_mxf_filename();
 394
 395         boost::filesystem::path video_to;
 396         video_to /= _film->dir (_film->dcp_name());
 397         video_to /= _film->video_mxf_filename ();
 398
 399         boost::system::error_code ec;
 400         boost::filesystem::create_hard_link (video_from, video_to, ec);
 401         if (ec) {
 402                 /* hard link failed; copy instead */
 403                 boost::filesystem::copy_file (video_from, video_to);
 404                 LOG_WARNING_NC ("Hard-link failed; fell back to copying");
 405         }
 406
 407         _picture_mxf->set_file (video_to);
 408
 409         /* Move the audio MXF into the DCP */
 410
 411         if (_sound_mxf) {
 412                 boost::filesystem::path audio_to;
 413                 audio_to /= _film->dir (_film->dcp_name ());
 414                 audio_to /= _film->audio_mxf_filename ();
 415
 416                 boost::filesystem::rename (_film->file (_film->audio_mxf_filename ()), audio_to, ec);
 417                 if (ec) {
 418                         throw FileError (
 419                                 String::compose (_("could not move audio MXF into the DCP (%1)"), ec.value ()), _film->file (_film->audio_mxf_filename ())
 420                                 );
 421                 }
 422
 423                 _sound_mxf->set_file (audio_to);
 424         }
 425
 426         dcp::DCP dcp (_film->dir (_film->dcp_name()));
 427
 428         shared_ptr<dcp::CPL> cpl (
 429                 new dcp::CPL (
 430                         _film->dcp_name(),
 431                         _film->dcp_content_type()->libdcp_kind ()
 432                         )
 433                 );
 434
 435         dcp.add (cpl);
 436
 437         shared_ptr<dcp::Reel> reel (new dcp::Reel ());
 438
 439         shared_ptr<dcp::MonoPictureMXF> mono = dynamic_pointer_cast<dcp::MonoPictureMXF> (_picture_mxf);
 440         if (mono) {
 441                 reel->add (shared_ptr<dcp::ReelPictureAsset> (new dcp::ReelMonoPictureAsset (mono, 0)));
 442                 dcp.add (mono);
 443         }
 444
 445         shared_ptr<dcp::StereoPictureMXF> stereo = dynamic_pointer_cast<dcp::StereoPictureMXF> (_picture_mxf);
 446         if (stereo) {
 447                 reel->add (shared_ptr<dcp::ReelPictureAsset> (new dcp::ReelStereoPictureAsset (stereo, 0)));
 448                 dcp.add (stereo);
 449         }
 450
 451         if (_sound_mxf) {
 452                 reel->add (shared_ptr<dcp::ReelSoundAsset> (new dcp::ReelSoundAsset (_sound_mxf, 0)));
 453                 dcp.add (_sound_mxf);
 454         }
 455
 456         cpl->add (reel);
 457
 458         shared_ptr<Job> job = _job.lock ();
 459         assert (job);
 460
 461         job->sub (_("Computing image digest"));
 462         _picture_mxf->hash (boost::bind (&Job::set_progress, job.get(), _1, false));
 463
 464         if (_sound_mxf) {
 465                 job->sub (_("Computing audio digest"));
 466                 _sound_mxf->hash (boost::bind (&Job::set_progress, job.get(), _1, false));
 467         }
 468
 469         dcp::XMLMetadata meta = Config::instance()->dcp_metadata ();
 470         meta.set_issue_date_now ();
 471
 472         dcp.write_xml (_film->interop () ? dcp::INTEROP : dcp::SMPTE, meta, _film->is_signed() ? make_signer () : shared_ptr<const dcp::Signer> ());
 473
 474         LOG_GENERAL (
 475                 N_("Wrote %1 FULL, %2 FAKE, %3 pushed to disk"), _full_written, _fake_written, _pushed_to_disk
 476                 );
 477 }
 478
 479 bool
 480 Writer::check_existing_picture_mxf_frame (FILE* mxf, int f, Eyes eyes)
 481 {
 482         /* Read the frame info as written */
 483         FILE* ifi = fopen_boost (_film->info_path (f, eyes), "r");
 484         if (!ifi) {
 485                 LOG_GENERAL ("Existing frame %1 has no info file", f);
 486                 return false;
 487         }
 488
 489         dcp::FrameInfo info (ifi);
 490         fclose (ifi);
 491         if (info.size == 0) {
 492                 LOG_GENERAL ("Existing frame %1 has no info file", f);
 493                 return false;
 494         }
 495
 496         /* Read the data from the MXF and hash it */
 497         dcpomatic_fseek (mxf, info.offset, SEEK_SET);
 498         EncodedData data (info.size);
 499         size_t const read = fread (data.data(), 1, data.size(), mxf);
 500         if (read != static_cast<size_t> (data.size ())) {
 501                 LOG_GENERAL ("Existing frame %1 is incomplete", f);
 502                 return false;
 503         }
 504
 505         MD5Digester digester;
 506         digester.add (data.data(), data.size());
 507         if (digester.get() != info.hash) {
 508                 LOG_GENERAL ("Existing frame %1 failed hash check", f);
 509                 return false;
 510         }
 511
 512         return true;
 513 }
 514
 515 void
 516 Writer::check_existing_picture_mxf ()
 517 {
 518         /* Try to open the existing MXF */
 519         boost::filesystem::path p;
 520         p /= _film->internal_video_mxf_dir ();
 521         p /= _film->internal_video_mxf_filename ();
 522         FILE* mxf = fopen_boost (p, "rb");
 523         if (!mxf) {
 524                 LOG_GENERAL ("Could not open existing MXF at %1 (errno=%2)", p.string(), errno);
 525                 return;
 526         }
 527
 528         int N = 0;
 529         for (boost::filesystem::directory_iterator i (_film->info_dir ()); i != boost::filesystem::directory_iterator (); ++i) {
 530                 ++N;
 531         }
 532
 533         while (true) {
 534
 535                 shared_ptr<Job> job = _job.lock ();
 536                 assert (job);
 537
 538                 if (N > 0) {
 539                         job->set_progress (float (_first_nonexistant_frame) / N);
 540                 }
 541
 542                 if (_film->three_d ()) {
 543                         if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_LEFT)) {
 544                                 break;
 545                         }
 546                         if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_RIGHT)) {
 547                                 break;
 548                         }
 549                 } else {
 550                         if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_BOTH)) {
 551                                 break;
 552                         }
 553                 }
 554
 555                 LOG_GENERAL ("Have existing frame %1", _first_nonexistant_frame);
 556                 ++_first_nonexistant_frame;
 557         }
 558
 559         fclose (mxf);
 560 }
 561
 562 /** @param frame Frame index.
 563  *  @return true if we can fake-write this frame.
 564  */
 565 bool
 566 Writer::can_fake_write (int frame) const
 567 {
 568         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 569            parameters in the MXF writer.
 570         */
 571         return (frame != 0 && frame < _first_nonexistant_frame);
 572 }
 573
 574 bool
 575 operator< (QueueItem const & a, QueueItem const & b)
 576 {
 577         if (a.frame != b.frame) {
 578                 return a.frame < b.frame;
 579         }
 580
 581         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 582 }
 583
 584 bool
 585 operator== (QueueItem const & a, QueueItem const & b)
 586 {
 587         return a.frame == b.frame && a.eyes == b.eyes;
 588 }