2 Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
23 #include "compose.hpp"
27 #include "dcpomatic_log.h"
28 #include "dcp_video.h"
29 #include "dcp_content_type.h"
30 #include "audio_mapping.h"
34 #include "audio_buffers.h"
36 #include "font_data.h"
38 #include "reel_writer.h"
39 #include "text_content.h"
41 #include <dcp/locale_convert.h>
42 #include <dcp/reel_file_asset.h>
51 /* OS X strikes again */
56 using std::dynamic_pointer_cast;
59 using std::make_shared;
64 using std::shared_ptr;
68 using boost::optional;
69 #if BOOST_VERSION >= 106100
70 using namespace boost::placeholders;
74 using namespace dcpomatic;
79 ignore_progress (float)
85 /** @param j Job to report progress to, or 0.
86 * @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
88 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
89 : WeakConstFilm (weak_film)
91 /* These will be reset to sensible values when J2KEncoder is created */
92 , _maximum_frames_in_memory (8)
93 , _maximum_queue_size (8)
94 , _text_only (text_only)
96 auto job = _job.lock ();
99 auto const reels = film()->reels();
100 for (auto p: reels) {
101 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
104 _last_written.resize (reels.size());
106 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
107 and captions arrive to the Writer in sequence. This is not so for video.
109 _audio_reel = _reels.begin ();
110 _subtitle_reel = _reels.begin ();
111 for (auto i: film()->closed_caption_tracks()) {
112 _caption_reels[i] = _reels.begin ();
114 _atmos_reel = _reels.begin ();
116 /* Check that the signer is OK */
118 if (!Config::instance()->signer_chain()->valid(&reason)) {
119 throw InvalidSignerError (reason);
128 _thread = boost::thread (boost::bind(&Writer::thread, this));
129 #ifdef DCPOMATIC_LINUX
130 pthread_setname_np (_thread.native_handle(), "writer");
139 terminate_thread (false);
144 /** Pass a video frame to the writer for writing to disk at some point.
145 * This method can be called with frames out of order.
146 * @param encoded JPEG2000-encoded data.
147 * @param frame Frame index within the DCP.
148 * @param eyes Eyes that this frame image is for.
151 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
153 boost::mutex::scoped_lock lock (_state_mutex);
155 while (_queued_full_in_memory > _maximum_frames_in_memory) {
156 /* There are too many full frames in memory; wake the main writer thread and
157 wait until it sorts everything out */
158 _empty_condition.notify_all ();
159 _full_condition.wait (lock);
163 qi.type = QueueItem::Type::FULL;
164 qi.encoded = encoded;
165 qi.reel = video_reel (frame);
166 qi.frame = frame - _reels[qi.reel].start ();
168 if (film()->three_d() && eyes == Eyes::BOTH) {
169 /* 2D material in a 3D DCP; fake the 3D */
170 qi.eyes = Eyes::LEFT;
171 _queue.push_back (qi);
172 ++_queued_full_in_memory;
173 qi.eyes = Eyes::RIGHT;
174 _queue.push_back (qi);
175 ++_queued_full_in_memory;
178 _queue.push_back (qi);
179 ++_queued_full_in_memory;
182 /* Now there's something to do: wake anything wait()ing on _empty_condition */
183 _empty_condition.notify_all ();
188 Writer::can_repeat (Frame frame) const
190 return frame > _reels[video_reel(frame)].start();
194 /** Repeat the last frame that was written to a reel as a new frame.
195 * @param frame Frame index within the DCP of the new (repeated) frame.
196 * @param eyes Eyes that this repeated frame image is for.
199 Writer::repeat (Frame frame, Eyes eyes)
201 boost::mutex::scoped_lock lock (_state_mutex);
203 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
204 /* The queue is too big, and the main writer thread can run and fix it, so
205 wake it and wait until it has done.
207 _empty_condition.notify_all ();
208 _full_condition.wait (lock);
212 qi.type = QueueItem::Type::REPEAT;
213 qi.reel = video_reel (frame);
214 qi.frame = frame - _reels[qi.reel].start ();
215 if (film()->three_d() && eyes == Eyes::BOTH) {
216 qi.eyes = Eyes::LEFT;
217 _queue.push_back (qi);
218 qi.eyes = Eyes::RIGHT;
219 _queue.push_back (qi);
222 _queue.push_back (qi);
225 /* Now there's something to do: wake anything wait()ing on _empty_condition */
226 _empty_condition.notify_all ();
231 Writer::fake_write (Frame frame, Eyes eyes)
233 boost::mutex::scoped_lock lock (_state_mutex);
235 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
236 /* The queue is too big, and the main writer thread can run and fix it, so
237 wake it and wait until it has done.
239 _empty_condition.notify_all ();
240 _full_condition.wait (lock);
243 size_t const reel = video_reel (frame);
244 Frame const frame_in_reel = frame - _reels[reel].start ();
247 qi.type = QueueItem::Type::FAKE;
250 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
251 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
255 qi.frame = frame_in_reel;
256 if (film()->three_d() && eyes == Eyes::BOTH) {
257 qi.eyes = Eyes::LEFT;
258 _queue.push_back (qi);
259 qi.eyes = Eyes::RIGHT;
260 _queue.push_back (qi);
263 _queue.push_back (qi);
266 /* Now there's something to do: wake anything wait()ing on _empty_condition */
267 _empty_condition.notify_all ();
271 /** Write some audio frames to the DCP.
272 * @param audio Audio data.
273 * @param time Time of this data within the DCP.
274 * This method is not thread safe.
277 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
279 DCPOMATIC_ASSERT (audio);
281 int const afr = film()->audio_frame_rate();
283 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
285 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
290 if (_audio_reel == _reels.end ()) {
291 /* This audio is off the end of the last reel; ignore it */
295 if (end <= _audio_reel->period().to) {
296 /* Easy case: we can write all the audio to this reel */
297 _audio_reel->write (audio);
299 } else if (_audio_reel->period().to <= t) {
300 /* This reel is entirely before the start of our audio; just skip the reel */
303 /* This audio is over a reel boundary; split the audio into two and write the first part */
304 DCPTime part_lengths[2] = {
305 _audio_reel->period().to - t,
306 end - _audio_reel->period().to
309 Frame part_frames[2] = {
310 part_lengths[0].frames_ceil(afr),
311 part_lengths[1].frames_ceil(afr)
314 if (part_frames[0]) {
315 shared_ptr<AudioBuffers> part (new AudioBuffers(audio, part_frames[0], 0));
316 _audio_reel->write (part);
319 if (part_frames[1]) {
320 audio.reset (new AudioBuffers(audio, part_frames[1], part_frames[0]));
326 t += part_lengths[0];
333 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
335 if (_atmos_reel->period().to == time) {
337 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
340 /* We assume that we get a video frame's worth of data here */
341 _atmos_reel->write (atmos, metadata);
345 /** Caller must hold a lock on _state_mutex */
347 Writer::have_sequenced_image_at_queue_head ()
349 if (_queue.empty ()) {
354 auto const & f = _queue.front();
355 return _last_written[f.reel].next(f);
360 Writer::LastWritten::next (QueueItem qi) const
362 if (qi.eyes == Eyes::BOTH) {
364 return qi.frame == (_frame + 1);
369 if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
373 if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
382 Writer::LastWritten::update (QueueItem qi)
395 boost::mutex::scoped_lock lock (_state_mutex);
399 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
400 /* We've got something to do: go and do it */
404 /* Nothing to do: wait until something happens which may indicate that we do */
405 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
406 _empty_condition.wait (lock);
407 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
410 /* We stop here if we have been asked to finish, and if either the queue
411 is empty or we do not have a sequenced image at its head (if this is the
412 case we will never terminate as no new frames will be sent once
415 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
416 /* (Hopefully temporarily) log anything that was not written */
417 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
418 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
419 for (auto const& i: _queue) {
420 if (i.type == QueueItem::Type::FULL) {
421 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
423 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
430 /* Write any frames that we can write; i.e. those that are in sequence. */
431 while (have_sequenced_image_at_queue_head ()) {
432 auto qi = _queue.front ();
433 _last_written[qi.reel].update (qi);
435 if (qi.type == QueueItem::Type::FULL && qi.encoded) {
436 --_queued_full_in_memory;
441 auto& reel = _reels[qi.reel];
444 case QueueItem::Type::FULL:
445 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
447 qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
449 reel.write (qi.encoded, qi.frame, qi.eyes);
452 case QueueItem::Type::FAKE:
453 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
454 reel.fake_write (qi.size);
457 case QueueItem::Type::REPEAT:
458 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
459 reel.repeat_write (qi.frame, qi.eyes);
465 _full_condition.notify_all ();
468 while (_queued_full_in_memory > _maximum_frames_in_memory) {
469 /* Too many frames in memory which can't yet be written to the stream.
470 Write some FULL frames to disk.
473 /* Find one from the back of the queue */
475 auto i = _queue.rbegin ();
476 while (i != _queue.rend() && (i->type != QueueItem::Type::FULL || !i->encoded)) {
480 DCPOMATIC_ASSERT (i != _queue.rend());
482 /* For the log message below */
483 int const awaiting = _last_written[_queue.front().reel].frame() + 1;
486 /* i is valid here, even though we don't hold a lock on the mutex,
487 since list iterators are unaffected by insertion and only this
488 thread could erase the last item in the list.
491 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
493 i->encoded->write_via_temp (
494 film()->j2c_path(i->reel, i->frame, i->eyes, true),
495 film()->j2c_path(i->reel, i->frame, i->eyes, false)
500 --_queued_full_in_memory;
501 _full_condition.notify_all ();
512 Writer::terminate_thread (bool can_throw)
514 boost::this_thread::disable_interruption dis;
516 boost::mutex::scoped_lock lock (_state_mutex);
519 _empty_condition.notify_all ();
520 _full_condition.notify_all ();
533 /** @param output_dcp Path to DCP folder to write */
535 Writer::finish (boost::filesystem::path output_dcp)
537 if (_thread.joinable()) {
538 LOG_GENERAL_NC ("Terminating writer thread");
539 terminate_thread (true);
542 LOG_GENERAL_NC ("Finishing ReelWriters");
544 for (auto& i: _reels) {
545 i.finish (output_dcp);
548 LOG_GENERAL_NC ("Writing XML");
550 dcp::DCP dcp (output_dcp);
552 auto cpl = make_shared<dcp::CPL>(
554 film()->dcp_content_type()->libdcp_kind()
559 /* Calculate digests for each reel in parallel */
561 auto job = _job.lock ();
563 job->sub (_("Computing digests"));
566 boost::asio::io_service service;
567 boost::thread_group pool;
569 auto work = make_shared<boost::asio::io_service::work>(service);
571 int const threads = max (1, Config::instance()->master_encoding_threads ());
573 for (int i = 0; i < threads; ++i) {
574 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
577 boost::function<void (float)> set_progress;
579 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
581 set_progress = &ignore_progress;
584 for (auto& i: _reels) {
585 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
587 service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
595 for (auto& i: _reels) {
596 cpl->add (i.create_reel(_reel_assets, _fonts, output_dcp, _have_subtitles, _have_closed_captions));
601 auto creator = Config::instance()->dcp_creator();
602 if (creator.empty()) {
603 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
606 auto issuer = Config::instance()->dcp_issuer();
607 if (issuer.empty()) {
608 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
611 cpl->set_ratings (film()->ratings());
613 vector<dcp::ContentVersion> cv;
614 for (auto i: film()->content_versions()) {
615 cv.push_back (dcp::ContentVersion(i));
617 cpl->set_content_versions (cv);
619 cpl->set_full_content_title_text (film()->name());
620 cpl->set_full_content_title_text_language (film()->name_language());
621 cpl->set_release_territory (film()->release_territory());
622 cpl->set_version_number (film()->version_number());
623 cpl->set_status (film()->status());
624 cpl->set_chain (film()->chain());
625 cpl->set_distributor (film()->distributor());
626 cpl->set_facility (film()->facility());
627 cpl->set_luminance (film()->luminance());
629 auto ac = film()->mapped_audio_channels();
630 dcp::MCASoundField field = (
631 find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSL)) != ac.end() ||
632 find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSR)) != ac.end()
633 ) ? dcp::MCASoundField::SEVEN_POINT_ONE : dcp::MCASoundField::FIVE_POINT_ONE;
635 dcp::MainSoundConfiguration msc (field, film()->audio_channels());
637 if (static_cast<int>(i) < film()->audio_channels()) {
638 msc.set_mapping (i, static_cast<dcp::Channel>(i));
642 cpl->set_main_sound_configuration (msc.to_string());
643 cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
644 cpl->set_main_picture_stored_area (film()->frame_size());
645 cpl->set_main_picture_active_area (film()->active_area());
647 vector<dcp::LanguageTag> sl = film()->subtitle_languages();
649 cpl->set_additional_subtitle_languages(std::vector<dcp::LanguageTag>(sl.begin() + 1, sl.end()));
652 auto signer = Config::instance()->signer_chain();
653 /* We did check earlier, but check again here to be on the safe side */
655 if (!signer->valid (&reason)) {
656 throw InvalidSignerError (reason);
660 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE,
663 dcp::LocalTime().as_string(),
666 Config::instance()->dcp_metadata_filename_format()
670 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
673 write_cover_sheet (output_dcp);
678 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
680 auto const cover = film()->file("COVER_SHEET.txt");
681 auto f = fopen_boost (cover, "w");
683 throw OpenFileError (cover, errno, OpenFileError::WRITE);
686 auto text = Config::instance()->cover_sheet ();
687 boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
688 boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
689 boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
690 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", film()->isdcf_metadata().audio_language);
692 auto subtitle_languages = film()->subtitle_languages();
693 if (subtitle_languages.empty()) {
694 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", "None");
696 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.front().description());
699 boost::uintmax_t size = 0;
701 auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
702 i != boost::filesystem::recursive_directory_iterator();
704 if (boost::filesystem::is_regular_file (i->path())) {
705 size += boost::filesystem::file_size (i->path());
709 if (size > (1000000000L)) {
710 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
712 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
715 auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
716 auto description = String::compose("%1.%2", ch.first, ch.second);
718 if (description == "0.0") {
719 description = _("None");
720 } else if (description == "1.0") {
721 description = _("Mono");
722 } else if (description == "2.0") {
723 description = _("Stereo");
725 boost::algorithm::replace_all (text, "$AUDIO", description);
728 film()->length().split(film()->video_frame_rate(), h, m, s, fr);
730 if (h == 0 && m == 0) {
731 length = String::compose("%1s", s);
732 } else if (h == 0 && m > 0) {
733 length = String::compose("%1m%2s", m, s);
734 } else if (h > 0 && m > 0) {
735 length = String::compose("%1h%2m%3s", h, m, s);
738 boost::algorithm::replace_all (text, "$LENGTH", length);
740 checked_fwrite (text.c_str(), text.length(), f, cover);
745 /** @param frame Frame index within the whole DCP.
746 * @return true if we can fake-write this frame.
749 Writer::can_fake_write (Frame frame) const
751 if (film()->encrypted()) {
752 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
756 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
757 parameters in the asset writer.
760 auto const & reel = _reels[video_reel(frame)];
762 /* Make frame relative to the start of the reel */
763 frame -= reel.start ();
764 return (frame != 0 && frame < reel.first_nonexistant_frame());
768 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
770 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
772 vector<ReelWriter>::iterator* reel = nullptr;
775 case TextType::OPEN_SUBTITLE:
776 reel = &_subtitle_reel;
777 _have_subtitles = true;
779 case TextType::CLOSED_CAPTION:
780 DCPOMATIC_ASSERT (track);
781 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
782 reel = &_caption_reels[*track];
783 _have_closed_captions.insert (*track);
786 DCPOMATIC_ASSERT (false);
789 DCPOMATIC_ASSERT (*reel != _reels.end());
790 while ((*reel)->period().to <= period.from) {
792 DCPOMATIC_ASSERT (*reel != _reels.end());
795 (*reel)->write (text, type, track, period);
800 Writer::write (vector<FontData> fonts)
802 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
804 for (auto const& i: fonts) {
806 for (auto& j: _fonts) {
813 _fonts.push_back (i);
820 operator< (QueueItem const & a, QueueItem const & b)
822 if (a.reel != b.reel) {
823 return a.reel < b.reel;
826 if (a.frame != b.frame) {
827 return a.frame < b.frame;
830 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
835 operator== (QueueItem const & a, QueueItem const & b)
837 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
842 Writer::set_encoder_threads (int threads)
844 boost::mutex::scoped_lock lm (_state_mutex);
845 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
846 _maximum_queue_size = threads * 16;
851 Writer::write (ReferencedReelAsset asset)
853 _reel_assets.push_back (asset);
858 Writer::video_reel (int frame) const
860 auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
862 while (i < _reels.size() && !_reels[i].period().contains (t)) {
866 DCPOMATIC_ASSERT (i < _reels.size ());
872 Writer::set_digest_progress (Job* job, float progress)
874 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
876 _digest_progresses[boost::this_thread::get_id()] = progress;
877 float min_progress = FLT_MAX;
878 for (auto const& i: _digest_progresses) {
879 min_progress = min (min_progress, i.second);
882 job->set_progress (min_progress);
889 /** Calculate hashes for any referenced MXF assets which do not already have one */
891 Writer::calculate_referenced_digests (boost::function<void (float)> set_progress)
893 for (auto const& i: _reel_assets) {
894 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
895 if (file && !file->hash()) {
896 file->asset_ref().asset()->hash (set_progress);
897 file->set_hash (file->asset_ref().asset()->hash());