2 Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 #include "compose.hpp"
26 #include "dcpomatic_log.h"
27 #include "dcp_video.h"
28 #include "dcp_content_type.h"
29 #include "audio_mapping.h"
33 #include "audio_buffers.h"
37 #include "reel_writer.h"
38 #include "text_content.h"
40 #include <dcp/locale_convert.h>
41 #include <boost/foreach.hpp>
49 /* OS X strikes again */
61 using boost::shared_ptr;
62 using boost::weak_ptr;
63 using boost::dynamic_pointer_cast;
64 using boost::optional;
65 #if BOOST_VERSION >= 106100
66 using namespace boost::placeholders;
69 using namespace dcpomatic;
71 Writer::Writer (shared_ptr<const Film> film, weak_ptr<Job> j)
75 , _queued_full_in_memory (0)
76 /* These will be reset to sensible values when J2KEncoder is created */
77 , _maximum_frames_in_memory (8)
78 , _maximum_queue_size (8)
84 shared_ptr<Job> job = _job.lock ();
85 DCPOMATIC_ASSERT (job);
88 list<DCPTimePeriod> const reels = _film->reels ();
89 BOOST_FOREACH (DCPTimePeriod p, reels) {
90 _reels.push_back (ReelWriter (film, p, job, reel_index++, reels.size(), _film->content_summary(p)));
93 _last_written.resize (reels.size());
95 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
96 and captions arrive to the Writer in sequence. This is not so for video.
98 _audio_reel = _reels.begin ();
99 _subtitle_reel = _reels.begin ();
100 BOOST_FOREACH (DCPTextTrack i, _film->closed_caption_tracks()) {
101 _caption_reels[i] = _reels.begin ();
103 _atmos_reel = _reels.begin ();
105 /* Check that the signer is OK */
107 if (!Config::instance()->signer_chain()->valid(&reason)) {
108 throw InvalidSignerError (reason);
115 _thread = boost::thread (boost::bind(&Writer::thread, this));
116 #ifdef DCPOMATIC_LINUX
117 pthread_setname_np (_thread.native_handle(), "writer");
123 terminate_thread (false);
126 /** Pass a video frame to the writer for writing to disk at some point.
127 * This method can be called with frames out of order.
128 * @param encoded JPEG2000-encoded data.
129 * @param frame Frame index within the DCP.
130 * @param eyes Eyes that this frame image is for.
133 Writer::write (Data encoded, Frame frame, Eyes eyes)
135 boost::mutex::scoped_lock lock (_state_mutex);
137 while (_queued_full_in_memory > _maximum_frames_in_memory) {
138 /* There are too many full frames in memory; wake the main writer thread and
139 wait until it sorts everything out */
140 _empty_condition.notify_all ();
141 _full_condition.wait (lock);
145 qi.type = QueueItem::FULL;
146 qi.encoded = encoded;
147 qi.reel = video_reel (frame);
148 qi.frame = frame - _reels[qi.reel].start ();
150 if (_film->three_d() && eyes == EYES_BOTH) {
151 /* 2D material in a 3D DCP; fake the 3D */
153 _queue.push_back (qi);
154 ++_queued_full_in_memory;
155 qi.eyes = EYES_RIGHT;
156 _queue.push_back (qi);
157 ++_queued_full_in_memory;
160 _queue.push_back (qi);
161 ++_queued_full_in_memory;
164 /* Now there's something to do: wake anything wait()ing on _empty_condition */
165 _empty_condition.notify_all ();
169 Writer::can_repeat (Frame frame) const
171 return frame > _reels[video_reel(frame)].start();
174 /** Repeat the last frame that was written to a reel as a new frame.
175 * @param frame Frame index within the DCP of the new (repeated) frame.
176 * @param eyes Eyes that this repeated frame image is for.
179 Writer::repeat (Frame frame, Eyes eyes)
181 boost::mutex::scoped_lock lock (_state_mutex);
183 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
184 /* The queue is too big, and the main writer thread can run and fix it, so
185 wake it and wait until it has done.
187 _empty_condition.notify_all ();
188 _full_condition.wait (lock);
192 qi.type = QueueItem::REPEAT;
193 qi.reel = video_reel (frame);
194 qi.frame = frame - _reels[qi.reel].start ();
195 if (_film->three_d() && eyes == EYES_BOTH) {
197 _queue.push_back (qi);
198 qi.eyes = EYES_RIGHT;
199 _queue.push_back (qi);
202 _queue.push_back (qi);
205 /* Now there's something to do: wake anything wait()ing on _empty_condition */
206 _empty_condition.notify_all ();
210 Writer::fake_write (Frame frame, Eyes eyes)
212 boost::mutex::scoped_lock lock (_state_mutex);
214 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
215 /* The queue is too big, and the main writer thread can run and fix it, so
216 wake it and wait until it has done.
218 _empty_condition.notify_all ();
219 _full_condition.wait (lock);
222 size_t const reel = video_reel (frame);
223 Frame const frame_in_reel = frame - _reels[reel].start ();
226 qi.type = QueueItem::FAKE;
229 shared_ptr<InfoFileHandle> info_file = _film->info_file_handle(_reels[reel].period(), true);
230 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
234 qi.frame = frame_in_reel;
235 if (_film->three_d() && eyes == EYES_BOTH) {
237 _queue.push_back (qi);
238 qi.eyes = EYES_RIGHT;
239 _queue.push_back (qi);
242 _queue.push_back (qi);
245 /* Now there's something to do: wake anything wait()ing on _empty_condition */
246 _empty_condition.notify_all ();
249 /** Write some audio frames to the DCP.
250 * @param audio Audio data.
251 * @param time Time of this data within the DCP.
252 * This method is not thread safe.
255 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
257 DCPOMATIC_ASSERT (audio);
259 int const afr = _film->audio_frame_rate();
261 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
263 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
268 if (_audio_reel == _reels.end ()) {
269 /* This audio is off the end of the last reel; ignore it */
273 if (end <= _audio_reel->period().to) {
274 /* Easy case: we can write all the audio to this reel */
275 _audio_reel->write (audio);
277 } else if (_audio_reel->period().to <= t) {
278 /* This reel is entirely before the start of our audio; just skip the reel */
281 /* This audio is over a reel boundary; split the audio into two and write the first part */
282 DCPTime part_lengths[2] = {
283 _audio_reel->period().to - t,
284 end - _audio_reel->period().to
287 Frame part_frames[2] = {
288 part_lengths[0].frames_ceil(afr),
289 part_lengths[1].frames_ceil(afr)
292 if (part_frames[0]) {
293 shared_ptr<AudioBuffers> part (new AudioBuffers(audio, part_frames[0], 0));
294 _audio_reel->write (part);
297 if (part_frames[1]) {
298 audio.reset (new AudioBuffers(audio, part_frames[1], part_frames[0]));
304 t += part_lengths[0];
311 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
313 if (_atmos_reel->period().to == time) {
315 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
318 /* We assume that we get a video frame's worth of data here */
319 _atmos_reel->write (atmos, metadata);
323 /** Caller must hold a lock on _state_mutex */
325 Writer::have_sequenced_image_at_queue_head ()
327 if (_queue.empty ()) {
332 QueueItem const & f = _queue.front();
333 return _last_written[f.reel].next(f);
338 Writer::LastWritten::next (QueueItem qi) const
340 if (qi.eyes == EYES_BOTH) {
342 return qi.frame == (_frame + 1);
347 if (_eyes == EYES_LEFT && qi.frame == _frame && qi.eyes == EYES_RIGHT) {
351 if (_eyes == EYES_RIGHT && qi.frame == (_frame + 1) && qi.eyes == EYES_LEFT) {
360 Writer::LastWritten::update (QueueItem qi)
373 boost::mutex::scoped_lock lock (_state_mutex);
377 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
378 /* We've got something to do: go and do it */
382 /* Nothing to do: wait until something happens which may indicate that we do */
383 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
384 _empty_condition.wait (lock);
385 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
388 if (_finish && _queue.empty()) {
392 /* We stop here if we have been asked to finish, and if either the queue
393 is empty or we do not have a sequenced image at its head (if this is the
394 case we will never terminate as no new frames will be sent once
397 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
398 /* (Hopefully temporarily) log anything that was not written */
399 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
400 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
401 BOOST_FOREACH (QueueItem const& i, _queue) {
402 if (i.type == QueueItem::FULL) {
403 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
405 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
412 /* Write any frames that we can write; i.e. those that are in sequence. */
413 while (have_sequenced_image_at_queue_head ()) {
414 QueueItem qi = _queue.front ();
415 _last_written[qi.reel].update (qi);
417 if (qi.type == QueueItem::FULL && qi.encoded) {
418 --_queued_full_in_memory;
423 ReelWriter& reel = _reels[qi.reel];
426 case QueueItem::FULL:
427 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
429 qi.encoded = Data (_film->j2c_path (qi.reel, qi.frame, qi.eyes, false));
431 reel.write (qi.encoded, qi.frame, qi.eyes);
434 case QueueItem::FAKE:
435 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
436 reel.fake_write (qi.size);
439 case QueueItem::REPEAT:
440 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
441 reel.repeat_write (qi.frame, qi.eyes);
447 _full_condition.notify_all ();
450 while (_queued_full_in_memory > _maximum_frames_in_memory) {
451 /* Too many frames in memory which can't yet be written to the stream.
452 Write some FULL frames to disk.
455 /* Find one from the back of the queue */
457 list<QueueItem>::reverse_iterator i = _queue.rbegin ();
458 while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
462 DCPOMATIC_ASSERT (i != _queue.rend());
464 /* For the log message below */
465 int const awaiting = _last_written[_queue.front().reel].frame() + 1;
468 /* i is valid here, even though we don't hold a lock on the mutex,
469 since list iterators are unaffected by insertion and only this
470 thread could erase the last item in the list.
473 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
475 i->encoded->write_via_temp (
476 _film->j2c_path (i->reel, i->frame, i->eyes, true),
477 _film->j2c_path (i->reel, i->frame, i->eyes, false)
482 --_queued_full_in_memory;
483 _full_condition.notify_all ();
493 Writer::terminate_thread (bool can_throw)
495 boost::this_thread::disable_interruption dis;
497 boost::mutex::scoped_lock lock (_state_mutex);
500 _empty_condition.notify_all ();
501 _full_condition.notify_all ();
516 if (!_thread.joinable()) {
520 LOG_GENERAL_NC ("Terminating writer thread");
522 terminate_thread (true);
524 LOG_GENERAL_NC ("Finishing ReelWriters");
526 BOOST_FOREACH (ReelWriter& i, _reels) {
530 LOG_GENERAL_NC ("Writing XML");
532 dcp::DCP dcp (_film->dir (_film->dcp_name()));
534 shared_ptr<dcp::CPL> cpl (
537 _film->dcp_content_type()->libdcp_kind ()
543 /* Calculate digests for each reel in parallel */
545 shared_ptr<Job> job = _job.lock ();
546 job->sub (_("Computing digests"));
548 boost::asio::io_service service;
549 boost::thread_group pool;
551 shared_ptr<boost::asio::io_service::work> work (new boost::asio::io_service::work (service));
553 int const threads = max (1, Config::instance()->master_encoding_threads ());
555 for (int i = 0; i < threads; ++i) {
556 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
559 BOOST_FOREACH (ReelWriter& i, _reels) {
560 boost::function<void (float)> set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
561 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
570 BOOST_FOREACH (ReelWriter& i, _reels) {
571 cpl->add (i.create_reel (_reel_assets, _fonts));
576 string creator = Config::instance()->dcp_creator();
577 if (creator.empty()) {
578 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
581 string issuer = Config::instance()->dcp_issuer();
582 if (issuer.empty()) {
583 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
586 cpl->set_ratings (_film->ratings());
588 vector<dcp::ContentVersion> cv;
589 BOOST_FOREACH (string i, _film->content_versions()) {
590 cv.push_back (dcp::ContentVersion(i));
592 cpl->set_content_versions (cv);
594 cpl->set_full_content_title_text (_film->name());
595 cpl->set_full_content_title_text_language (_film->name_language());
596 cpl->set_release_territory (_film->release_territory());
597 cpl->set_version_number (_film->version_number());
598 cpl->set_status (_film->status());
599 cpl->set_chain (_film->chain());
600 cpl->set_distributor (_film->distributor());
601 cpl->set_facility (_film->facility());
602 cpl->set_luminance (_film->luminance());
604 list<int> ac = _film->mapped_audio_channels ();
605 dcp::MCASoundField field = (
606 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSL)) != ac.end() ||
607 find(ac.begin(), ac.end(), static_cast<int>(dcp::BSR)) != ac.end()
608 ) ? dcp::SEVEN_POINT_ONE : dcp::FIVE_POINT_ONE;
610 dcp::MainSoundConfiguration msc (field, _film->audio_channels());
611 BOOST_FOREACH (int i, ac) {
612 if (i < _film->audio_channels()) {
613 msc.set_mapping (i, static_cast<dcp::Channel>(i));
617 cpl->set_main_sound_configuration (msc.to_string());
618 cpl->set_main_sound_sample_rate (_film->audio_frame_rate());
619 cpl->set_main_picture_stored_area (_film->frame_size());
620 cpl->set_main_picture_active_area (_film->active_area());
622 shared_ptr<const dcp::CertificateChain> signer;
623 signer = Config::instance()->signer_chain ();
624 /* We did check earlier, but check again here to be on the safe side */
626 if (!signer->valid (&reason)) {
627 throw InvalidSignerError (reason);
631 _film->interop() ? dcp::INTEROP : dcp::SMPTE,
634 dcp::LocalTime().as_string(),
635 String::compose("Created by libdcp %1", dcp::version),
637 Config::instance()->dcp_metadata_filename_format()
641 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
644 write_cover_sheet ();
648 Writer::write_cover_sheet ()
650 boost::filesystem::path const cover = _film->file ("COVER_SHEET.txt");
651 FILE* f = fopen_boost (cover, "w");
653 throw OpenFileError (cover, errno, OpenFileError::WRITE);
656 string text = Config::instance()->cover_sheet ();
657 boost::algorithm::replace_all (text, "$CPL_NAME", _film->name());
658 boost::algorithm::replace_all (text, "$TYPE", _film->dcp_content_type()->pretty_name());
659 boost::algorithm::replace_all (text, "$CONTAINER", _film->container()->container_nickname());
660 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _film->isdcf_metadata().audio_language);
662 optional<string> subtitle_language;
663 BOOST_FOREACH (shared_ptr<Content> i, _film->content()) {
664 BOOST_FOREACH (shared_ptr<TextContent> j, i->text) {
665 if (j->type() == TEXT_OPEN_SUBTITLE && j->use()) {
666 subtitle_language = j->language ();
670 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_language.get_value_or("None"));
672 boost::uintmax_t size = 0;
674 boost::filesystem::recursive_directory_iterator i = boost::filesystem::recursive_directory_iterator(_film->dir(_film->dcp_name()));
675 i != boost::filesystem::recursive_directory_iterator();
677 if (boost::filesystem::is_regular_file (i->path ())) {
678 size += boost::filesystem::file_size (i->path ());
682 if (size > (1000000000L)) {
683 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1GB", dcp::locale_convert<string> (size / 1000000000.0, 1, true)));
685 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1MB", dcp::locale_convert<string> (size / 1000000.0, 1, true)));
688 pair<int, int> ch = audio_channel_types (_film->mapped_audio_channels(), _film->audio_channels());
689 string description = String::compose("%1.%2", ch.first, ch.second);
691 if (description == "0.0") {
692 description = _("None");
693 } else if (description == "1.0") {
694 description = _("Mono");
695 } else if (description == "2.0") {
696 description = _("Stereo");
698 boost::algorithm::replace_all (text, "$AUDIO", description);
701 _film->length().split (_film->video_frame_rate(), h, m, s, fr);
703 if (h == 0 && m == 0) {
704 length = String::compose("%1s", s);
705 } else if (h == 0 && m > 0) {
706 length = String::compose("%1m%2s", m, s);
707 } else if (h > 0 && m > 0) {
708 length = String::compose("%1h%2m%3s", h, m, s);
711 boost::algorithm::replace_all (text, "$LENGTH", length);
713 checked_fwrite (text.c_str(), text.length(), f, cover);
717 /** @param frame Frame index within the whole DCP.
718 * @return true if we can fake-write this frame.
721 Writer::can_fake_write (Frame frame) const
723 if (_film->encrypted()) {
724 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
728 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
729 parameters in the asset writer.
732 ReelWriter const & reel = _reels[video_reel(frame)];
734 /* Make frame relative to the start of the reel */
735 frame -= reel.start ();
736 return (frame != 0 && frame < reel.first_nonexistant_frame());
739 /** @param track Closed caption track if type == TEXT_CLOSED_CAPTION */
741 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
743 vector<ReelWriter>::iterator* reel = 0;
746 case TEXT_OPEN_SUBTITLE:
747 reel = &_subtitle_reel;
749 case TEXT_CLOSED_CAPTION:
750 DCPOMATIC_ASSERT (track);
751 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
752 reel = &_caption_reels[*track];
755 DCPOMATIC_ASSERT (false);
758 DCPOMATIC_ASSERT (*reel != _reels.end());
759 while ((*reel)->period().to <= period.from) {
761 DCPOMATIC_ASSERT (*reel != _reels.end());
764 (*reel)->write (text, type, track, period);
768 Writer::write (list<shared_ptr<Font> > fonts)
770 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
772 BOOST_FOREACH (shared_ptr<Font> i, fonts) {
774 BOOST_FOREACH (shared_ptr<Font> j, _fonts) {
781 _fonts.push_back (i);
787 operator< (QueueItem const & a, QueueItem const & b)
789 if (a.reel != b.reel) {
790 return a.reel < b.reel;
793 if (a.frame != b.frame) {
794 return a.frame < b.frame;
797 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
801 operator== (QueueItem const & a, QueueItem const & b)
803 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
807 Writer::set_encoder_threads (int threads)
809 boost::mutex::scoped_lock lm (_state_mutex);
810 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
811 _maximum_queue_size = threads * 16;
815 Writer::write (ReferencedReelAsset asset)
817 _reel_assets.push_back (asset);
821 Writer::video_reel (int frame) const
823 DCPTime t = DCPTime::from_frames (frame, _film->video_frame_rate ());
825 while (i < _reels.size() && !_reels[i].period().contains (t)) {
829 DCPOMATIC_ASSERT (i < _reels.size ());
834 Writer::set_digest_progress (Job* job, float progress)
836 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
838 _digest_progresses[boost::this_thread::get_id()] = progress;
839 float min_progress = FLT_MAX;
840 for (map<boost::thread::id, float>::const_iterator i = _digest_progresses.begin(); i != _digest_progresses.end(); ++i) {
841 min_progress = min (min_progress, i->second);
844 job->set_progress (min_progress);