2 Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 #include "compose.hpp"
26 #include "dcpomatic_log.h"
27 #include "dcp_video.h"
28 #include "dcp_content_type.h"
29 #include "audio_mapping.h"
33 #include "audio_buffers.h"
37 #include "reel_writer.h"
38 #include "text_content.h"
40 #include <dcp/locale_convert.h>
41 #include <boost/foreach.hpp>
49 /* OS X strikes again */
61 using boost::shared_ptr;
62 using boost::weak_ptr;
63 using boost::dynamic_pointer_cast;
64 using boost::optional;
65 #if BOOST_VERSION >= 106100
66 using namespace boost::placeholders;
70 using namespace dcpomatic;
72 Writer::Writer (shared_ptr<const Film> film, weak_ptr<Job> j)
76 , _queued_full_in_memory (0)
77 /* These will be reset to sensible values when J2KEncoder is created */
78 , _maximum_frames_in_memory (8)
79 , _maximum_queue_size (8)
85 shared_ptr<Job> job = _job.lock ();
86 DCPOMATIC_ASSERT (job);
89 list<DCPTimePeriod> const reels = _film->reels ();
90 BOOST_FOREACH (DCPTimePeriod p, reels) {
91 _reels.push_back (ReelWriter (film, p, job, reel_index++, reels.size(), _film->content_summary(p)));
94 _last_written.resize (reels.size());
96 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
97 and captions arrive to the Writer in sequence. This is not so for video.
99 _audio_reel = _reels.begin ();
100 _subtitle_reel = _reels.begin ();
101 BOOST_FOREACH (DCPTextTrack i, _film->closed_caption_tracks()) {
102 _caption_reels[i] = _reels.begin ();
104 _atmos_reel = _reels.begin ();
106 /* Check that the signer is OK */
108 if (!Config::instance()->signer_chain()->valid(&reason)) {
109 throw InvalidSignerError (reason);
116 _thread = boost::thread (boost::bind(&Writer::thread, this));
117 #ifdef DCPOMATIC_LINUX
118 pthread_setname_np (_thread.native_handle(), "writer");
124 terminate_thread (false);
127 /** Pass a video frame to the writer for writing to disk at some point.
128 * This method can be called with frames out of order.
129 * @param encoded JPEG2000-encoded data.
130 * @param frame Frame index within the DCP.
131 * @param eyes Eyes that this frame image is for.
134 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
136 boost::mutex::scoped_lock lock (_state_mutex);
138 while (_queued_full_in_memory > _maximum_frames_in_memory) {
139 /* There are too many full frames in memory; wake the main writer thread and
140 wait until it sorts everything out */
141 _empty_condition.notify_all ();
142 _full_condition.wait (lock);
146 qi.type = QueueItem::FULL;
147 qi.encoded = encoded;
148 qi.reel = video_reel (frame);
149 qi.frame = frame - _reels[qi.reel].start ();
151 if (_film->three_d() && eyes == EYES_BOTH) {
152 /* 2D material in a 3D DCP; fake the 3D */
154 _queue.push_back (qi);
155 ++_queued_full_in_memory;
156 qi.eyes = EYES_RIGHT;
157 _queue.push_back (qi);
158 ++_queued_full_in_memory;
161 _queue.push_back (qi);
162 ++_queued_full_in_memory;
165 /* Now there's something to do: wake anything wait()ing on _empty_condition */
166 _empty_condition.notify_all ();
170 Writer::can_repeat (Frame frame) const
172 return frame > _reels[video_reel(frame)].start();
175 /** Repeat the last frame that was written to a reel as a new frame.
176 * @param frame Frame index within the DCP of the new (repeated) frame.
177 * @param eyes Eyes that this repeated frame image is for.
180 Writer::repeat (Frame frame, Eyes eyes)
182 boost::mutex::scoped_lock lock (_state_mutex);
184 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
185 /* The queue is too big, and the main writer thread can run and fix it, so
186 wake it and wait until it has done.
188 _empty_condition.notify_all ();
189 _full_condition.wait (lock);
193 qi.type = QueueItem::REPEAT;
194 qi.reel = video_reel (frame);
195 qi.frame = frame - _reels[qi.reel].start ();
196 if (_film->three_d() && eyes == EYES_BOTH) {
198 _queue.push_back (qi);
199 qi.eyes = EYES_RIGHT;
200 _queue.push_back (qi);
203 _queue.push_back (qi);
206 /* Now there's something to do: wake anything wait()ing on _empty_condition */
207 _empty_condition.notify_all ();
211 Writer::fake_write (Frame frame, Eyes eyes)
213 boost::mutex::scoped_lock lock (_state_mutex);
215 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
216 /* The queue is too big, and the main writer thread can run and fix it, so
217 wake it and wait until it has done.
219 _empty_condition.notify_all ();
220 _full_condition.wait (lock);
223 size_t const reel = video_reel (frame);
224 Frame const frame_in_reel = frame - _reels[reel].start ();
227 qi.type = QueueItem::FAKE;
230 shared_ptr<InfoFileHandle> info_file = _film->info_file_handle(_reels[reel].period(), true);
231 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
235 qi.frame = frame_in_reel;
236 if (_film->three_d() && eyes == EYES_BOTH) {
238 _queue.push_back (qi);
239 qi.eyes = EYES_RIGHT;
240 _queue.push_back (qi);
243 _queue.push_back (qi);
246 /* Now there's something to do: wake anything wait()ing on _empty_condition */
247 _empty_condition.notify_all ();
250 /** Write some audio frames to the DCP.
251 * @param audio Audio data.
252 * @param time Time of this data within the DCP.
253 * This method is not thread safe.
256 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
258 DCPOMATIC_ASSERT (audio);
260 int const afr = _film->audio_frame_rate();
262 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
264 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
269 if (_audio_reel == _reels.end ()) {
270 /* This audio is off the end of the last reel; ignore it */
274 if (end <= _audio_reel->period().to) {
275 /* Easy case: we can write all the audio to this reel */
276 _audio_reel->write (audio);
278 } else if (_audio_reel->period().to <= t) {
279 /* This reel is entirely before the start of our audio; just skip the reel */
282 /* This audio is over a reel boundary; split the audio into two and write the first part */
283 DCPTime part_lengths[2] = {
284 _audio_reel->period().to - t,
285 end - _audio_reel->period().to
288 Frame part_frames[2] = {
289 part_lengths[0].frames_ceil(afr),
290 part_lengths[1].frames_ceil(afr)
293 if (part_frames[0]) {
294 shared_ptr<AudioBuffers> part (new AudioBuffers(audio, part_frames[0], 0));
295 _audio_reel->write (part);
298 if (part_frames[1]) {
299 audio.reset (new AudioBuffers(audio, part_frames[1], part_frames[0]));
305 t += part_lengths[0];
312 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
314 if (_atmos_reel->period().to == time) {
316 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
319 /* We assume that we get a video frame's worth of data here */
320 _atmos_reel->write (atmos, metadata);
324 /** Caller must hold a lock on _state_mutex */
326 Writer::have_sequenced_image_at_queue_head ()
328 if (_queue.empty ()) {
333 QueueItem const & f = _queue.front();
334 return _last_written[f.reel].next(f);
339 Writer::LastWritten::next (QueueItem qi) const
341 if (qi.eyes == EYES_BOTH) {
343 return qi.frame == (_frame + 1);
348 if (_eyes == EYES_LEFT && qi.frame == _frame && qi.eyes == EYES_RIGHT) {
352 if (_eyes == EYES_RIGHT && qi.frame == (_frame + 1) && qi.eyes == EYES_LEFT) {
361 Writer::LastWritten::update (QueueItem qi)
374 boost::mutex::scoped_lock lock (_state_mutex);
378 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
379 /* We've got something to do: go and do it */
383 /* Nothing to do: wait until something happens which may indicate that we do */
384 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
385 _empty_condition.wait (lock);
386 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
389 if (_finish && _queue.empty()) {
393 /* We stop here if we have been asked to finish, and if either the queue
394 is empty or we do not have a sequenced image at its head (if this is the
395 case we will never terminate as no new frames will be sent once
398 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
399 /* (Hopefully temporarily) log anything that was not written */
400 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
401 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
402 BOOST_FOREACH (QueueItem const& i, _queue) {
403 if (i.type == QueueItem::FULL) {
404 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
406 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
413 /* Write any frames that we can write; i.e. those that are in sequence. */
414 while (have_sequenced_image_at_queue_head ()) {
415 QueueItem qi = _queue.front ();
416 _last_written[qi.reel].update (qi);
418 if (qi.type == QueueItem::FULL && qi.encoded) {
419 --_queued_full_in_memory;
424 ReelWriter& reel = _reels[qi.reel];
427 case QueueItem::FULL:
428 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
430 qi.encoded.reset (new ArrayData(_film->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
432 reel.write (qi.encoded, qi.frame, qi.eyes);
435 case QueueItem::FAKE:
436 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
437 reel.fake_write (qi.size);
440 case QueueItem::REPEAT:
441 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
442 reel.repeat_write (qi.frame, qi.eyes);
448 _full_condition.notify_all ();
451 while (_queued_full_in_memory > _maximum_frames_in_memory) {
452 /* Too many frames in memory which can't yet be written to the stream.
453 Write some FULL frames to disk.
456 /* Find one from the back of the queue */
458 list<QueueItem>::reverse_iterator i = _queue.rbegin ();
459 while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
463 DCPOMATIC_ASSERT (i != _queue.rend());
465 /* For the log message below */
466 int const awaiting = _last_written[_queue.front().reel].frame() + 1;
469 /* i is valid here, even though we don't hold a lock on the mutex,
470 since list iterators are unaffected by insertion and only this
471 thread could erase the last item in the list.
474 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
476 i->encoded->write_via_temp (
477 _film->j2c_path (i->reel, i->frame, i->eyes, true),
478 _film->j2c_path (i->reel, i->frame, i->eyes, false)
483 --_queued_full_in_memory;
484 _full_condition.notify_all ();
494 Writer::terminate_thread (bool can_throw)
496 boost::this_thread::disable_interruption dis;
498 boost::mutex::scoped_lock lock (_state_mutex);
501 _empty_condition.notify_all ();
502 _full_condition.notify_all ();
517 if (!_thread.joinable()) {
521 LOG_GENERAL_NC ("Terminating writer thread");
523 terminate_thread (true);
525 LOG_GENERAL_NC ("Finishing ReelWriters");
527 BOOST_FOREACH (ReelWriter& i, _reels) {
531 LOG_GENERAL_NC ("Writing XML");
533 dcp::DCP dcp (_film->dir (_film->dcp_name()));
535 shared_ptr<dcp::CPL> cpl (
538 _film->dcp_content_type()->libdcp_kind ()
544 /* Calculate digests for each reel in parallel */
546 shared_ptr<Job> job = _job.lock ();
547 job->sub (_("Computing digests"));
549 boost::asio::io_service service;
550 boost::thread_group pool;
552 shared_ptr<boost::asio::io_service::work> work (new boost::asio::io_service::work (service));
554 int const threads = max (1, Config::instance()->master_encoding_threads ());
556 for (int i = 0; i < threads; ++i) {
557 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
560 BOOST_FOREACH (ReelWriter& i, _reels) {
561 boost::function<void (float)> set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
562 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
571 BOOST_FOREACH (ReelWriter& i, _reels) {
572 cpl->add (i.create_reel (_reel_assets, _fonts));
577 string creator = Config::instance()->dcp_creator();
578 if (creator.empty()) {
579 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
582 string issuer = Config::instance()->dcp_issuer();
583 if (issuer.empty()) {
584 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
587 cpl->set_ratings (_film->ratings());
589 vector<dcp::ContentVersion> cv;
590 BOOST_FOREACH (string i, _film->content_versions()) {
591 cv.push_back (dcp::ContentVersion(i));
593 cpl->set_content_versions (cv);
595 cpl->set_full_content_title_text (_film->name());
596 cpl->set_full_content_title_text_language (_film->name_language());
597 cpl->set_release_territory (_film->release_territory());
598 cpl->set_version_number (_film->version_number());
599 cpl->set_status (_film->status());
600 cpl->set_chain (_film->chain());
601 cpl->set_distributor (_film->distributor());
602 cpl->set_facility (_film->facility());
603 cpl->set_luminance (_film->luminance());
605 list<int> ac = _film->mapped_audio_channels ();
606 dcp::MCASoundField field = (
607 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSL)) != ac.end() ||
608 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSR)) != ac.end()
609 ) ? dcp::SEVEN_POINT_ONE : dcp::FIVE_POINT_ONE;
611 dcp::MainSoundConfiguration msc (field, _film->audio_channels());
612 BOOST_FOREACH (int i, ac) {
613 if (i < _film->audio_channels()) {
614 msc.set_mapping (i, static_cast<dcp::Channel>(i));
618 cpl->set_main_sound_configuration (msc.to_string());
619 cpl->set_main_sound_sample_rate (_film->audio_frame_rate());
620 cpl->set_main_picture_stored_area (_film->frame_size());
621 cpl->set_main_picture_active_area (_film->active_area());
623 shared_ptr<const dcp::CertificateChain> signer;
624 signer = Config::instance()->signer_chain ();
625 /* We did check earlier, but check again here to be on the safe side */
627 if (!signer->valid (&reason)) {
628 throw InvalidSignerError (reason);
632 _film->interop() ? dcp::INTEROP : dcp::SMPTE,
635 dcp::LocalTime().as_string(),
636 String::compose("Created by libdcp %1", dcp::version),
638 Config::instance()->dcp_metadata_filename_format()
642 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
645 write_cover_sheet ();
649 Writer::write_cover_sheet ()
651 boost::filesystem::path const cover = _film->file ("COVER_SHEET.txt");
652 FILE* f = fopen_boost (cover, "w");
654 throw OpenFileError (cover, errno, OpenFileError::WRITE);
657 string text = Config::instance()->cover_sheet ();
658 boost::algorithm::replace_all (text, "$CPL_NAME", _film->name());
659 boost::algorithm::replace_all (text, "$TYPE", _film->dcp_content_type()->pretty_name());
660 boost::algorithm::replace_all (text, "$CONTAINER", _film->container()->container_nickname());
661 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _film->isdcf_metadata().audio_language);
663 vector<dcp::LanguageTag> subtitle_languages = _film->subtitle_languages();
664 if (subtitle_languages.empty()) {
665 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", "None");
667 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.front().description());
670 boost::uintmax_t size = 0;
672 boost::filesystem::recursive_directory_iterator i = boost::filesystem::recursive_directory_iterator(_film->dir(_film->dcp_name()));
673 i != boost::filesystem::recursive_directory_iterator();
675 if (boost::filesystem::is_regular_file (i->path ())) {
676 size += boost::filesystem::file_size (i->path ());
680 if (size > (1000000000L)) {
681 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1GB", dcp::locale_convert<string> (size / 1000000000.0, 1, true)));
683 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1MB", dcp::locale_convert<string> (size / 1000000.0, 1, true)));
686 pair<int, int> ch = audio_channel_types (_film->mapped_audio_channels(), _film->audio_channels());
687 string description = String::compose("%1.%2", ch.first, ch.second);
689 if (description == "0.0") {
690 description = _("None");
691 } else if (description == "1.0") {
692 description = _("Mono");
693 } else if (description == "2.0") {
694 description = _("Stereo");
696 boost::algorithm::replace_all (text, "$AUDIO", description);
699 _film->length().split (_film->video_frame_rate(), h, m, s, fr);
701 if (h == 0 && m == 0) {
702 length = String::compose("%1s", s);
703 } else if (h == 0 && m > 0) {
704 length = String::compose("%1m%2s", m, s);
705 } else if (h > 0 && m > 0) {
706 length = String::compose("%1h%2m%3s", h, m, s);
709 boost::algorithm::replace_all (text, "$LENGTH", length);
711 checked_fwrite (text.c_str(), text.length(), f, cover);
715 /** @param frame Frame index within the whole DCP.
716 * @return true if we can fake-write this frame.
719 Writer::can_fake_write (Frame frame) const
721 if (_film->encrypted()) {
722 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
726 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
727 parameters in the asset writer.
730 ReelWriter const & reel = _reels[video_reel(frame)];
732 /* Make frame relative to the start of the reel */
733 frame -= reel.start ();
734 return (frame != 0 && frame < reel.first_nonexistant_frame());
737 /** @param track Closed caption track if type == TEXT_CLOSED_CAPTION */
739 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
741 vector<ReelWriter>::iterator* reel = 0;
744 case TEXT_OPEN_SUBTITLE:
745 reel = &_subtitle_reel;
747 case TEXT_CLOSED_CAPTION:
748 DCPOMATIC_ASSERT (track);
749 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
750 reel = &_caption_reels[*track];
753 DCPOMATIC_ASSERT (false);
756 DCPOMATIC_ASSERT (*reel != _reels.end());
757 while ((*reel)->period().to <= period.from) {
759 DCPOMATIC_ASSERT (*reel != _reels.end());
762 (*reel)->write (text, type, track, period);
766 Writer::write (list<shared_ptr<Font> > fonts)
768 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
770 BOOST_FOREACH (shared_ptr<Font> i, fonts) {
772 BOOST_FOREACH (shared_ptr<Font> j, _fonts) {
779 _fonts.push_back (i);
785 operator< (QueueItem const & a, QueueItem const & b)
787 if (a.reel != b.reel) {
788 return a.reel < b.reel;
791 if (a.frame != b.frame) {
792 return a.frame < b.frame;
795 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
799 operator== (QueueItem const & a, QueueItem const & b)
801 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
805 Writer::set_encoder_threads (int threads)
807 boost::mutex::scoped_lock lm (_state_mutex);
808 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
809 _maximum_queue_size = threads * 16;
813 Writer::write (ReferencedReelAsset asset)
815 _reel_assets.push_back (asset);
819 Writer::video_reel (int frame) const
821 DCPTime t = DCPTime::from_frames (frame, _film->video_frame_rate ());
823 while (i < _reels.size() && !_reels[i].period().contains (t)) {
827 DCPOMATIC_ASSERT (i < _reels.size ());
832 Writer::set_digest_progress (Job* job, float progress)
834 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
836 _digest_progresses[boost::this_thread::get_id()] = progress;
837 float min_progress = FLT_MAX;
838 for (map<boost::thread::id, float>::const_iterator i = _digest_progresses.begin(); i != _digest_progresses.end(); ++i) {
839 min_progress = min (min_progress, i->second);
842 job->set_progress (min_progress);