2 Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 /** @file src/j2k_encoder.cc
23 * @brief J2K encoder class.
27 #include "compose.hpp"
30 #include "dcp_video.h"
31 #include "dcpomatic_log.h"
32 #include "encode_server_description.h"
33 #include "encode_server_finder.h"
35 #include "j2k_encoder.h"
37 #include "player_video.h"
40 #include <libcxml/cxml.h>
49 using std::make_shared;
50 using std::shared_ptr;
52 using boost::optional;
54 using namespace dcpomatic;
56 static grk_plugin::GrokInitializer grokInitializer;
58 /** @param film Film that we are encoding.
59 * @param writer Writer that we are using.
61 J2KEncoder::J2KEncoder(shared_ptr<const Film> film, Writer& writer)
65 , _dcpomatic_context(film, writer, _history, Config::instance()->gpu_binary_location())
66 , _context(Config::instance()->enable_gpu() ? new grk_plugin::GrokContext(_dcpomatic_context) : nullptr)
68 servers_list_changed ();
72 J2KEncoder::~J2KEncoder ()
74 _server_found_connection.disconnect();
77 boost::mutex::scoped_lock lm (_threads_mutex);
87 _server_found_connection = EncodeServerFinder::instance()->ServersListChanged.connect(
88 boost::bind(&J2KEncoder::servers_list_changed, this)
96 if (Config::instance()->enable_gpu()) {
102 void J2KEncoder::resume()
104 if (Config::instance()->enable_gpu()) {
105 _context = new grk_plugin::GrokContext(_dcpomatic_context);
106 servers_list_changed();
112 J2KEncoder::end (bool isFinal)
115 boost::mutex::scoped_lock lock (_queue_mutex);
117 LOG_GENERAL (N_("Clearing queue of %1"), _queue.size ());
119 /* Keep waking workers until the queue is empty */
120 while (!_queue.empty ()) {
122 _empty_condition.notify_all ();
123 _full_condition.wait (lock);
128 LOG_GENERAL_NC (N_("Terminating encoder threads"));
131 boost::mutex::scoped_lock lm (_threads_mutex);
132 terminate_threads ();
135 /* Something might have been thrown during terminate_threads */
138 LOG_GENERAL (N_("Mopping up %1"), _queue.size());
140 /* The following sequence of events can occur in the above code:
141 1. a remote worker takes the last image off the queue
142 2. the loop above terminates
143 3. the remote worker fails to encode the image and puts it back on the queue
144 4. the remote worker is then terminated by terminate_threads
146 So just mop up anything left in the queue here.
149 for (auto & i: _queue) {
150 if (Config::instance()->enable_gpu ()) {
151 if (!_context->scheduleCompress(i)){
152 LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), i.index());
157 LOG_GENERAL(N_("Encode left-over frame %1"), i.index());
160 make_shared<dcp::ArrayData>(i.encode_locally()),
165 } catch (std::exception& e) {
166 LOG_ERROR (N_("Local encode failed (%1)"), e.what ());
176 /** @return an estimate of the current number of frames we are encoding per second,
180 J2KEncoder::current_encoding_rate () const
182 return _history.rate ();
186 /** @return Number of video frames that have been queued for encoding */
188 J2KEncoder::video_frames_enqueued () const
190 if (!_last_player_video_time) {
194 return _last_player_video_time->frames_floor (_film->video_frame_rate ());
198 /** Should be called when a frame has been encoded successfully */
200 J2KEncoder::frame_done ()
206 /** Called to request encoding of the next video frame in the DCP. This is called in order,
207 * so each time the supplied frame is the one after the previous one.
208 * pv represents one video frame, and could be empty if there is nothing to encode
209 * for this DCP frame.
211 * @param pv PlayerVideo to encode.
212 * @param time Time of \p pv within the DCP.
215 J2KEncoder::encode (shared_ptr<PlayerVideo> pv, DCPTime time)
221 boost::mutex::scoped_lock lm (_threads_mutex);
223 threads = _threads->size();
225 threads = std::thread::hardware_concurrency();
228 boost::mutex::scoped_lock queue_lock (_queue_mutex);
230 /* Wait until the queue has gone down a bit. Allow one thing in the queue even
231 when there are no threads.
233 while (_queue.size() >= (threads * 2) + 1) {
234 LOG_TIMING ("decoder-sleep queue=%1 threads=%2", _queue.size(), threads);
235 _full_condition.wait (queue_lock);
236 LOG_TIMING ("decoder-wake queue=%1 threads=%2", _queue.size(), threads);
240 /* Re-throw any exception raised by one of our threads. If more
241 than one has thrown an exception, only one will be rethrown, I think;
242 but then, if that happens something has gone badly wrong.
246 auto const position = time.frames_floor(_film->video_frame_rate());
248 if (_writer.can_fake_write(position)) {
249 /* We can fake-write this frame */
250 LOG_DEBUG_ENCODE("Frame @ %1 FAKE", to_string(time));
251 _writer.fake_write(position, pv->eyes ());
253 } else if (pv->has_j2k() && !_film->reencode_j2k()) {
254 LOG_DEBUG_ENCODE("Frame @ %1 J2K", to_string(time));
255 /* This frame already has J2K data, so just write it */
256 _writer.write(pv->j2k(), position, pv->eyes ());
258 } else if (_last_player_video[pv->eyes()] && _writer.can_repeat(position) && pv->same(_last_player_video[pv->eyes()])) {
259 LOG_DEBUG_ENCODE("Frame @ %1 REPEAT", to_string(time));
260 _writer.repeat(position, pv->eyes());
262 LOG_DEBUG_ENCODE("Frame @ %1 ENCODE", to_string(time));
263 /* Queue this new frame for encoding */
264 LOG_TIMING ("add-frame-to-queue queue=%1", _queue.size ());
265 auto dcpv = DCPVideo(
268 _film->video_frame_rate(),
269 _film->j2k_bandwidth(),
272 _queue.push_back (dcpv);
274 /* The queue might not be empty any more, so notify anything which is
277 _empty_condition.notify_all ();
280 _last_player_video[pv->eyes()] = pv;
281 _last_player_video_time = time;
285 /** Caller must hold a lock on _threads_mutex */
287 J2KEncoder::terminate_threads ()
289 boost::this_thread::disable_interruption dis;
295 _threads->interrupt_all ();
297 _threads->join_all ();
298 } catch (exception& e) {
299 LOG_ERROR ("join() threw an exception: %1", e.what());
301 LOG_ERROR_NC ("join() threw an exception");
309 J2KEncoder::encoder_thread (optional<EncodeServerDescription> server)
312 auto config = Config::instance ();
314 start_of_thread ("J2KEncoder");
317 LOG_TIMING ("start-encoder-thread thread=%1 server=%2", thread_id (), server->host_name ());
319 LOG_TIMING ("start-encoder-thread thread=%1 server=localhost", thread_id ());
322 /* Number of seconds that we currently wait between attempts
323 to connect to the server; not relevant for localhost
326 int remote_backoff = 0;
330 LOG_TIMING ("encoder-sleep thread=%1", thread_id ());
331 boost::mutex::scoped_lock lock (_queue_mutex);
332 while (_queue.empty ()) {
333 _empty_condition.wait (lock);
336 LOG_TIMING ("encoder-wake thread=%1 queue=%2", thread_id(), _queue.size());
337 auto vf = _queue.front ();
339 /* We're about to commit to either encoding this frame or putting it back onto the queue,
340 so we must not be interrupted until one or other of these things have happened. This
341 block has thread interruption disabled.
344 boost::this_thread::disable_interruption dis;
346 LOG_TIMING ("encoder-pop thread=%1 frame=%2 eyes=%3", thread_id(), vf.index(), static_cast<int>(vf.eyes()));
351 shared_ptr<Data> encoded;
353 /* We need to encode this input */
356 encoded = make_shared<dcp::ArrayData>(vf.encode_remotely(server.get()));
358 if (remote_backoff > 0) {
359 LOG_GENERAL ("%1 was lost, but now she is found; removing backoff", server->host_name ());
362 /* This job succeeded, so remove any backoff */
365 } catch (std::exception& e) {
366 if (remote_backoff < 60) {
368 remote_backoff += 10;
371 N_("Remote encode of %1 on %2 failed (%3); thread sleeping for %4s"),
372 vf.index(), server->host_name(), e.what(), remote_backoff
378 if (!_context->launch(vf, config->selected_gpu()) || !_context->scheduleCompress(vf)) {
379 LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index());
380 _queue.push_front (vf);
385 LOG_TIMING ("start-local-encode thread=%1 frame=%2", thread_id(), vf.index());
386 encoded = make_shared<dcp::ArrayData>(vf.encode_locally());
387 LOG_TIMING ("finish-local-encode thread=%1 frame=%2", thread_id(), vf.index());
388 } catch (std::exception& e) {
389 /* This is very bad, so don't cope with it, just pass it on */
390 LOG_ERROR (N_("Local encode failed (%1)"), e.what ());
397 _writer.write(encoded, vf.index(), vf.eyes());
400 if (!Config::instance()->enable_gpu ()) {
402 LOG_GENERAL (N_("[%1] J2KEncoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf.index());
403 _queue.push_front (vf);
409 if (remote_backoff > 0) {
410 boost::this_thread::sleep (boost::posix_time::seconds (remote_backoff));
413 /* The queue might not be full any more, so notify anything that is waiting on that */
415 _full_condition.notify_all ();
418 catch (boost::thread_interrupted& e) {
419 /* Ignore these and just stop the thread */
420 _full_condition.notify_all ();
425 /* Wake anything waiting on _full_condition so it can see the exception */
426 _full_condition.notify_all ();
431 J2KEncoder::servers_list_changed ()
433 boost::mutex::scoped_lock lm (_threads_mutex);
435 terminate_threads ();
436 _threads = make_shared<boost::thread_group>();
438 /* XXX: could re-use threads */
440 if (!Config::instance()->only_servers_encode ()) {
441 for (int i = 0; i < Config::instance()->master_encoding_threads (); ++i) {
442 #ifdef DCPOMATIC_LINUX
443 auto t = _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, optional<EncodeServerDescription>()));
444 pthread_setname_np (t->native_handle(), "encode-worker");
446 _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, optional<EncodeServerDescription>()));
451 for (auto i: EncodeServerFinder::instance()->servers()) {
452 if (!i.current_link_version()) {
456 LOG_GENERAL (N_("Adding %1 worker threads for remote %2"), i.threads(), i.host_name ());
457 for (int j = 0; j < i.threads(); ++j) {
458 _threads->create_thread(boost::bind(&J2KEncoder::encoder_thread, this, i));
462 _writer.set_encoder_threads(_threads->size());