2 Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 #include "compose.hpp"
26 #include "dcpomatic_log.h"
27 #include "dcp_video.h"
28 #include "dcp_content_type.h"
29 #include "audio_mapping.h"
33 #include "audio_buffers.h"
37 #include "reel_writer.h"
38 #include "text_content.h"
40 #include <dcp/locale_convert.h>
41 #include <boost/foreach.hpp>
49 /* OS X strikes again */
61 using boost::shared_ptr;
62 using boost::weak_ptr;
63 using boost::dynamic_pointer_cast;
64 using boost::optional;
67 Writer::Writer (shared_ptr<const Film> film, weak_ptr<Job> j)
72 , _queued_full_in_memory (0)
73 /* These will be reset to sensible values when J2KEncoder is created */
74 , _maximum_frames_in_memory (8)
75 , _maximum_queue_size (8)
81 shared_ptr<Job> job = _job.lock ();
82 DCPOMATIC_ASSERT (job);
85 list<DCPTimePeriod> const reels = _film->reels ();
86 BOOST_FOREACH (DCPTimePeriod p, reels) {
87 _reels.push_back (ReelWriter (film, p, job, reel_index++, reels.size(), _film->content_summary(p)));
90 /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
91 and captions arrive to the Writer in sequence. This is not so for video.
93 _audio_reel = _reels.begin ();
94 _subtitle_reel = _reels.begin ();
95 BOOST_FOREACH (DCPTextTrack i, _film->closed_caption_tracks()) {
96 _caption_reels[i] = _reels.begin ();
99 /* Check that the signer is OK if we need one */
101 if (_film->is_signed() && !Config::instance()->signer_chain()->valid(&reason)) {
102 throw InvalidSignerError (reason);
109 _thread = new boost::thread (boost::bind (&Writer::thread, this));
110 #ifdef DCPOMATIC_LINUX
111 pthread_setname_np (_thread->native_handle(), "writer");
117 terminate_thread (false);
120 /** Pass a video frame to the writer for writing to disk at some point.
121 * This method can be called with frames out of order.
122 * @param encoded JPEG2000-encoded data.
123 * @param frame Frame index within the DCP.
124 * @param eyes Eyes that this frame image is for.
127 Writer::write (Data encoded, Frame frame, Eyes eyes)
129 boost::mutex::scoped_lock lock (_state_mutex);
131 while (_queued_full_in_memory > _maximum_frames_in_memory) {
132 /* There are too many full frames in memory; wake the main writer thread and
133 wait until it sorts everything out */
134 _empty_condition.notify_all ();
135 _full_condition.wait (lock);
139 qi.type = QueueItem::FULL;
140 qi.encoded = encoded;
141 qi.reel = video_reel (frame);
142 qi.frame = frame - _reels[qi.reel].start ();
144 if (_film->three_d() && eyes == EYES_BOTH) {
145 /* 2D material in a 3D DCP; fake the 3D */
147 _queue.push_back (qi);
148 ++_queued_full_in_memory;
149 qi.eyes = EYES_RIGHT;
150 _queue.push_back (qi);
151 ++_queued_full_in_memory;
154 _queue.push_back (qi);
155 ++_queued_full_in_memory;
158 /* Now there's something to do: wake anything wait()ing on _empty_condition */
159 _empty_condition.notify_all ();
163 Writer::can_repeat (Frame frame) const
165 return frame > _reels[video_reel(frame)].start();
168 /** Repeat the last frame that was written to a reel as a new frame.
169 * @param frame Frame index within the DCP of the new (repeated) frame.
170 * @param eyes Eyes that this repeated frame image is for.
173 Writer::repeat (Frame frame, Eyes eyes)
175 boost::mutex::scoped_lock lock (_state_mutex);
177 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
178 /* The queue is too big, and the main writer thread can run and fix it, so
179 wake it and wait until it has done.
181 _empty_condition.notify_all ();
182 _full_condition.wait (lock);
186 qi.type = QueueItem::REPEAT;
187 qi.reel = video_reel (frame);
188 qi.frame = frame - _reels[qi.reel].start ();
189 if (_film->three_d() && eyes == EYES_BOTH) {
191 _queue.push_back (qi);
192 qi.eyes = EYES_RIGHT;
193 _queue.push_back (qi);
196 _queue.push_back (qi);
199 /* Now there's something to do: wake anything wait()ing on _empty_condition */
200 _empty_condition.notify_all ();
204 Writer::fake_write (Frame frame, Eyes eyes)
206 boost::mutex::scoped_lock lock (_state_mutex);
208 while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
209 /* The queue is too big, and the main writer thread can run and fix it, so
210 wake it and wait until it has done.
212 _empty_condition.notify_all ();
213 _full_condition.wait (lock);
216 size_t const reel = video_reel (frame);
217 Frame const reel_frame = frame - _reels[reel].start ();
219 FILE* file = fopen_boost (_film->info_file(_reels[reel].period()), "rb");
221 throw ReadFileError (_film->info_file(_reels[reel].period()));
223 dcp::FrameInfo info = _reels[reel].read_frame_info (file, reel_frame, eyes);
227 qi.type = QueueItem::FAKE;
230 qi.frame = reel_frame;
231 if (_film->three_d() && eyes == EYES_BOTH) {
233 _queue.push_back (qi);
234 qi.eyes = EYES_RIGHT;
235 _queue.push_back (qi);
238 _queue.push_back (qi);
241 /* Now there's something to do: wake anything wait()ing on _empty_condition */
242 _empty_condition.notify_all ();
245 /** Write some audio frames to the DCP.
246 * @param audio Audio data.
247 * @param time Time of this data within the DCP.
248 * This method is not thread safe.
251 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
253 DCPOMATIC_ASSERT (audio);
255 int const afr = _film->audio_frame_rate();
257 DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
259 /* The audio we get might span a reel boundary, and if so we have to write it in bits */
264 if (_audio_reel == _reels.end ()) {
265 /* This audio is off the end of the last reel; ignore it */
269 if (end <= _audio_reel->period().to) {
270 /* Easy case: we can write all the audio to this reel */
271 _audio_reel->write (audio);
274 /* Split the audio into two and write the first part */
275 DCPTime part_lengths[2] = {
276 _audio_reel->period().to - t,
277 end - _audio_reel->period().to
280 Frame part_frames[2] = {
281 part_lengths[0].frames_ceil(afr),
282 part_lengths[1].frames_ceil(afr)
285 if (part_frames[0]) {
286 shared_ptr<AudioBuffers> part (new AudioBuffers (audio->channels(), part_frames[0]));
287 part->copy_from (audio.get(), part_frames[0], 0, 0);
288 _audio_reel->write (part);
291 if (part_frames[1]) {
292 shared_ptr<AudioBuffers> part (new AudioBuffers (audio->channels(), part_frames[1]));
293 part->copy_from (audio.get(), part_frames[1], part_frames[0], 0);
300 t += part_lengths[0];
305 /** This must be called from Writer::thread() with an appropriate lock held */
307 Writer::have_sequenced_image_at_queue_head ()
309 if (_queue.empty ()) {
315 QueueItem const & f = _queue.front();
316 ReelWriter const & reel = _reels[f.reel];
318 /* The queue should contain only EYES_LEFT/EYES_RIGHT pairs or EYES_BOTH */
320 if (f.eyes == EYES_BOTH) {
322 return f.frame == (reel.last_written_video_frame() + 1);
327 if (reel.last_written_eyes() == EYES_LEFT && f.frame == reel.last_written_video_frame() && f.eyes == EYES_RIGHT) {
331 if (reel.last_written_eyes() == EYES_RIGHT && f.frame == (reel.last_written_video_frame() + 1) && f.eyes == EYES_LEFT) {
344 boost::mutex::scoped_lock lock (_state_mutex);
348 if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
349 /* We've got something to do: go and do it */
353 /* Nothing to do: wait until something happens which may indicate that we do */
354 LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
355 _empty_condition.wait (lock);
356 LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
359 if (_finish && _queue.empty()) {
363 /* We stop here if we have been asked to finish, and if either the queue
364 is empty or we do not have a sequenced image at its head (if this is the
365 case we will never terminate as no new frames will be sent once
368 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
369 /* (Hopefully temporarily) log anything that was not written */
370 if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
371 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
372 for (list<QueueItem>::const_iterator i = _queue.begin(); i != _queue.end(); ++i) {
373 if (i->type == QueueItem::FULL) {
374 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i->frame, (int) i->eyes);
376 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i->size, i->frame, (int) i->eyes);
383 /* Write any frames that we can write; i.e. those that are in sequence. */
384 while (have_sequenced_image_at_queue_head ()) {
385 QueueItem qi = _queue.front ();
387 if (qi.type == QueueItem::FULL && qi.encoded) {
388 --_queued_full_in_memory;
393 ReelWriter& reel = _reels[qi.reel];
396 case QueueItem::FULL:
397 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
399 qi.encoded = Data (_film->j2c_path (qi.reel, qi.frame, qi.eyes, false));
401 reel.write (qi.encoded, qi.frame, qi.eyes);
404 case QueueItem::FAKE:
405 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
406 reel.fake_write (qi.frame, qi.eyes, qi.size);
409 case QueueItem::REPEAT:
410 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
411 reel.repeat_write (qi.frame, qi.eyes);
417 _full_condition.notify_all ();
420 while (_queued_full_in_memory > _maximum_frames_in_memory) {
421 /* Too many frames in memory which can't yet be written to the stream.
422 Write some FULL frames to disk.
425 /* Find one from the back of the queue */
427 list<QueueItem>::reverse_iterator i = _queue.rbegin ();
428 while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
432 DCPOMATIC_ASSERT (i != _queue.rend());
434 /* For the log message below */
435 int const awaiting = _reels[_queue.front().reel].last_written_video_frame() + 1;
438 /* i is valid here, even though we don't hold a lock on the mutex,
439 since list iterators are unaffected by insertion and only this
440 thread could erase the last item in the list.
443 LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
445 i->encoded->write_via_temp (
446 _film->j2c_path (i->reel, i->frame, i->eyes, true),
447 _film->j2c_path (i->reel, i->frame, i->eyes, false)
452 --_queued_full_in_memory;
453 _full_condition.notify_all ();
463 Writer::terminate_thread (bool can_throw)
465 boost::mutex::scoped_lock lock (_state_mutex);
471 _empty_condition.notify_all ();
472 _full_condition.notify_all ();
475 if (_thread->joinable ()) {
494 LOG_GENERAL_NC ("Terminating writer thread");
496 terminate_thread (true);
498 LOG_GENERAL_NC ("Finishing ReelWriters");
500 BOOST_FOREACH (ReelWriter& i, _reels) {
504 LOG_GENERAL_NC ("Writing XML");
506 dcp::DCP dcp (_film->dir (_film->dcp_name()));
508 shared_ptr<dcp::CPL> cpl (
511 _film->dcp_content_type()->libdcp_kind ()
517 /* Calculate digests for each reel in parallel */
519 shared_ptr<Job> job = _job.lock ();
520 job->sub (_("Computing digests"));
522 boost::asio::io_service service;
523 boost::thread_group pool;
525 shared_ptr<boost::asio::io_service::work> work (new boost::asio::io_service::work (service));
527 int const threads = max (1, Config::instance()->master_encoding_threads ());
529 for (int i = 0; i < threads; ++i) {
530 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
533 BOOST_FOREACH (ReelWriter& i, _reels) {
534 boost::function<void (float)> set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
535 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
542 /* Add reels to CPL */
544 BOOST_FOREACH (ReelWriter& i, _reels) {
545 cpl->add (i.create_reel (_reel_assets, _fonts));
548 dcp::XMLMetadata meta;
549 meta.annotation_text = cpl->annotation_text ();
550 meta.creator = Config::instance()->dcp_creator ();
551 if (meta.creator.empty ()) {
552 meta.creator = String::compose ("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
554 meta.issuer = Config::instance()->dcp_issuer ();
555 if (meta.issuer.empty ()) {
556 meta.issuer = String::compose ("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
558 meta.set_issue_date_now ();
560 cpl->set_metadata (meta);
562 shared_ptr<const dcp::CertificateChain> signer;
563 if (_film->is_signed ()) {
564 signer = Config::instance()->signer_chain ();
565 /* We did check earlier, but check again here to be on the safe side */
567 if (!signer->valid (&reason)) {
568 throw InvalidSignerError (reason);
572 dcp.write_xml (_film->interop () ? dcp::INTEROP : dcp::SMPTE, meta, signer, Config::instance()->dcp_metadata_filename_format());
575 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
578 write_cover_sheet ();
582 Writer::write_cover_sheet ()
584 boost::filesystem::path const cover = _film->file ("COVER_SHEET.txt");
585 FILE* f = fopen_boost (cover, "w");
587 throw OpenFileError (cover, errno, false);
590 string text = Config::instance()->cover_sheet ();
591 boost::algorithm::replace_all (text, "$CPL_NAME", _film->name());
592 boost::algorithm::replace_all (text, "$TYPE", _film->dcp_content_type()->pretty_name());
593 boost::algorithm::replace_all (text, "$CONTAINER", _film->container()->container_nickname());
594 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _film->isdcf_metadata().audio_language);
596 optional<string> subtitle_language;
597 BOOST_FOREACH (shared_ptr<Content> i, _film->content()) {
598 BOOST_FOREACH (shared_ptr<TextContent> j, i->text) {
599 if (j->type() == TEXT_OPEN_SUBTITLE && j->use()) {
600 subtitle_language = j->language ();
604 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_language.get_value_or("None"));
606 boost::uintmax_t size = 0;
608 boost::filesystem::recursive_directory_iterator i = boost::filesystem::recursive_directory_iterator(_film->dir(_film->dcp_name()));
609 i != boost::filesystem::recursive_directory_iterator();
611 if (boost::filesystem::is_regular_file (i->path ())) {
612 size += boost::filesystem::file_size (i->path ());
616 if (size > (1000000000L)) {
617 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1GB", dcp::locale_convert<string> (size / 1000000000.0, 1, true)));
619 boost::algorithm::replace_all (text, "$SIZE", String::compose ("%1MB", dcp::locale_convert<string> (size / 1000000.0, 1, true)));
622 pair<int, int> ch = audio_channel_types (_film->mapped_audio_channels(), _film->audio_channels());
623 string description = String::compose("%1.%2", ch.first, ch.second);
625 if (description == "0.0") {
626 description = _("None");
627 } else if (description == "1.0") {
628 description = _("Mono");
629 } else if (description == "2.0") {
630 description = _("Stereo");
632 boost::algorithm::replace_all (text, "$AUDIO", description);
635 _film->length().split (_film->video_frame_rate(), h, m, s, fr);
637 if (h == 0 && m == 0) {
638 length = String::compose("%1s", s);
639 } else if (h == 0 && m > 0) {
640 length = String::compose("%1m%2s", m, s);
641 } else if (h > 0 && m > 0) {
642 length = String::compose("%1h%2m%3s", h, m, s);
645 boost::algorithm::replace_all (text, "$LENGTH", length);
647 checked_fwrite (text.c_str(), text.length(), f, cover);
651 /** @param frame Frame index within the whole DCP.
652 * @return true if we can fake-write this frame.
655 Writer::can_fake_write (Frame frame) const
657 if (_film->encrypted()) {
658 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
662 /* We have to do a proper write of the first frame so that we can set up the JPEG2000
663 parameters in the asset writer.
666 ReelWriter const & reel = _reels[video_reel(frame)];
668 /* Make frame relative to the start of the reel */
669 frame -= reel.start ();
670 return (frame != 0 && frame < reel.first_nonexistant_frame());
673 /** @param track Closed caption track if type == TEXT_CLOSED_CAPTION */
675 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
677 vector<ReelWriter>::iterator* reel = 0;
680 case TEXT_OPEN_SUBTITLE:
681 reel = &_subtitle_reel;
683 case TEXT_CLOSED_CAPTION:
684 DCPOMATIC_ASSERT (track);
685 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
686 reel = &_caption_reels[*track];
689 DCPOMATIC_ASSERT (false);
692 DCPOMATIC_ASSERT (*reel != _reels.end());
693 while ((*reel)->period().to <= period.from) {
695 DCPOMATIC_ASSERT (*reel != _reels.end());
698 (*reel)->write (text, type, track, period);
702 Writer::write (list<shared_ptr<Font> > fonts)
704 /* Just keep a list of unique fonts and we'll deal with them in ::finish */
706 BOOST_FOREACH (shared_ptr<Font> i, fonts) {
708 BOOST_FOREACH (shared_ptr<Font> j, _fonts) {
715 _fonts.push_back (i);
721 operator< (QueueItem const & a, QueueItem const & b)
723 if (a.reel != b.reel) {
724 return a.reel < b.reel;
727 if (a.frame != b.frame) {
728 return a.frame < b.frame;
731 return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
735 operator== (QueueItem const & a, QueueItem const & b)
737 return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
741 Writer::set_encoder_threads (int threads)
743 boost::mutex::scoped_lock lm (_state_mutex);
744 _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
745 _maximum_queue_size = threads * 16;
749 Writer::write (ReferencedReelAsset asset)
751 _reel_assets.push_back (asset);
755 Writer::video_reel (int frame) const
757 DCPTime t = DCPTime::from_frames (frame, _film->video_frame_rate ());
759 while (i < _reels.size() && !_reels[i].period().contains (t)) {
763 DCPOMATIC_ASSERT (i < _reels.size ());
768 Writer::set_digest_progress (Job* job, float progress)
770 /* I believe this is thread-safe */
771 _digest_progresses[boost::this_thread::get_id()] = progress;
773 boost::mutex::scoped_lock lm (_digest_progresses_mutex);
774 float min_progress = FLT_MAX;
775 for (map<boost::thread::id, float>::const_iterator i = _digest_progresses.begin(); i != _digest_progresses.end(); ++i) {
776 min_progress = min (min_progress, i->second);
779 job->set_progress (min_progress);