2 Copyright (C) 2012-2015 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
21 /** @file src/encoder.h
22 * @brief Parent class for classes which can encode video and audio frames.
30 #include "dcp_video.h"
33 #include "encode_server_finder.h"
35 #include "player_video.h"
36 #include "encode_server_description.h"
37 #include "compose.hpp"
38 #include <libcxml/cxml.h>
39 #include <boost/foreach.hpp>
44 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
45 #define LOG_GENERAL_NC(...) _film->log()->log (__VA_ARGS__, LogEntry::TYPE_GENERAL);
46 #define LOG_ERROR(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_ERROR);
47 #define LOG_TIMING(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_TIMING);
51 using boost::shared_ptr;
52 using boost::weak_ptr;
53 using boost::optional;
56 int const Encoder::_history_size = 200;
58 /** @param f Film that we are encoding */
59 Encoder::Encoder (shared_ptr<const Film> film, shared_ptr<Writer> writer)
63 servers_list_changed ();
71 /* Destructors must not throw exceptions; anything bad
72 happening now is too late to worry about anyway,
81 weak_ptr<Encoder> wp = shared_from_this ();
82 _server_found_connection = EncodeServerFinder::instance()->ServersListChanged.connect (
83 boost::bind (&Encoder::call_servers_list_changed, wp)
87 /* We don't want the servers-list-changed callback trying to do things
88 during destruction of Encoder, and I think this is the neatest way
92 Encoder::call_servers_list_changed (weak_ptr<Encoder> encoder)
94 shared_ptr<Encoder> e = encoder.lock ();
96 e->servers_list_changed ();
103 boost::mutex::scoped_lock lock (_queue_mutex);
105 LOG_GENERAL (N_("Clearing queue of %1"), _queue.size ());
107 /* Keep waking workers until the queue is empty */
108 while (!_queue.empty ()) {
110 _empty_condition.notify_all ();
111 _full_condition.wait (lock);
116 LOG_GENERAL_NC (N_("Terminating encoder threads"));
118 terminate_threads ();
120 LOG_GENERAL (N_("Mopping up %1"), _queue.size());
122 /* The following sequence of events can occur in the above code:
123 1. a remote worker takes the last image off the queue
124 2. the loop above terminates
125 3. the remote worker fails to encode the image and puts it back on the queue
126 4. the remote worker is then terminated by terminate_threads
128 So just mop up anything left in the queue here.
131 for (list<shared_ptr<DCPVideo> >::iterator i = _queue.begin(); i != _queue.end(); ++i) {
132 LOG_GENERAL (N_("Encode left-over frame %1"), (*i)->index ());
135 (*i)->encode_locally (boost::bind (&Log::dcp_log, _film->log().get(), _1, _2)),
140 } catch (std::exception& e) {
141 LOG_ERROR (N_("Local encode failed (%1)"), e.what ());
146 /** @return an estimate of the current number of frames we are encoding per second,
150 Encoder::current_encoding_rate () const
152 boost::mutex::scoped_lock lock (_state_mutex);
153 if (int (_time_history.size()) < _history_size) {
158 gettimeofday (&now, 0);
160 return _history_size / (seconds (now) - seconds (_time_history.back ()));
163 /** @return Number of video frames that have been queued for encoding */
165 Encoder::video_frames_enqueued () const
167 if (!_last_player_video) {
171 return _last_player_video->time().frames_floor (_film->video_frame_rate ());
174 /** Should be called when a frame has been encoded successfully.
175 * @param n Source frame index.
178 Encoder::frame_done ()
180 boost::mutex::scoped_lock lock (_state_mutex);
183 gettimeofday (&tv, 0);
184 _time_history.push_front (tv);
185 if (int (_time_history.size()) > _history_size) {
186 _time_history.pop_back ();
190 /** Called to start encoding of the next video frame in the DCP. This is called in order,
191 * so each time the supplied frame is the one after the previous one.
192 * pv represents one video frame, and could be empty if there is nothing to encode
193 * for this DCP frame.
196 Encoder::encode (shared_ptr<PlayerVideo> pv)
202 boost::mutex::scoped_lock threads_lock (_threads_mutex);
203 threads = _threads.size ();
206 boost::mutex::scoped_lock queue_lock (_queue_mutex);
208 /* Wait until the queue has gone down a bit. Allow one thing in the queue even
209 when there are no threads.
211 while (_queue.size() >= (threads * 2) + 1) {
212 LOG_TIMING ("decoder-sleep queue=%1 threads=%2", _queue.size(), threads);
213 _full_condition.wait (queue_lock);
214 LOG_TIMING ("decoder-wake queue=%1 threads=%2", _queue.size(), threads);
218 /* Re-throw any exception raised by one of our threads. If more
219 than one has thrown an exception, only one will be rethrown, I think;
220 but then, if that happens something has gone badly wrong.
224 Frame const position = pv->time().frames_floor(_film->video_frame_rate());
226 if (_writer->can_fake_write (position)) {
227 /* We can fake-write this frame */
228 _writer->fake_write (position, pv->eyes ());
230 } else if (pv->has_j2k ()) {
231 /* This frame already has JPEG2000 data, so just write it */
232 _writer->write (pv->j2k(), position, pv->eyes ());
233 } else if (_last_player_video && _writer->can_repeat(position) && pv->same (_last_player_video)) {
234 _writer->repeat (position, pv->eyes ());
236 /* Queue this new frame for encoding */
237 LOG_TIMING ("add-frame-to-queue queue=%1", _queue.size ());
238 _queue.push_back (shared_ptr<DCPVideo> (
242 _film->video_frame_rate(),
243 _film->j2k_bandwidth(),
249 /* The queue might not be empty any more, so notify anything which is
252 _empty_condition.notify_all ();
255 _last_player_video = pv;
259 Encoder::terminate_threads ()
261 boost::mutex::scoped_lock threads_lock (_threads_mutex);
264 for (list<boost::thread *>::iterator i = _threads.begin(); i != _threads.end(); ++i) {
265 LOG_GENERAL ("Terminating thread %1 of %2", n + 1, _threads.size ());
267 DCPOMATIC_ASSERT ((*i)->joinable ());
270 } catch (boost::thread_interrupted& e) {
271 /* This is to be expected */
274 LOG_GENERAL_NC ("Thread terminated");
282 Encoder::encoder_thread (optional<EncodeServerDescription> server)
286 LOG_TIMING ("start-encoder-thread thread=%1 server=%2", thread_id (), server->host_name ());
288 LOG_TIMING ("start-encoder-thread thread=%1 server=localhost", thread_id ());
291 /* Number of seconds that we currently wait between attempts
292 to connect to the server; not relevant for localhost
295 int remote_backoff = 0;
299 LOG_TIMING ("encoder-sleep thread=%1", thread_id ());
300 boost::mutex::scoped_lock lock (_queue_mutex);
301 while (_queue.empty ()) {
302 _empty_condition.wait (lock);
305 LOG_TIMING ("encoder-wake thread=%1 queue=%2", thread_id(), _queue.size());
306 shared_ptr<DCPVideo> vf = _queue.front ();
308 /* We're about to commit to either encoding this frame or putting it back onto the queue,
309 so we must not be interrupted until one or other of these things have happened. This
310 block has thread interruption disabled.
313 boost::this_thread::disable_interruption dis;
315 LOG_TIMING ("encoder-pop thread=%1 frame=%2 eyes=%3", thread_id(), vf->index(), (int) vf->eyes ());
320 optional<Data> encoded;
322 /* We need to encode this input */
325 encoded = vf->encode_remotely (server.get ());
327 if (remote_backoff > 0) {
328 LOG_GENERAL ("%1 was lost, but now she is found; removing backoff", server->host_name ());
331 /* This job succeeded, so remove any backoff */
334 } catch (std::exception& e) {
335 if (remote_backoff < 60) {
337 remote_backoff += 10;
340 N_("Remote encode of %1 on %2 failed (%3); thread sleeping for %4s"),
341 vf->index(), server->host_name(), e.what(), remote_backoff
347 LOG_TIMING ("start-local-encode thread=%1 frame=%2", thread_id(), vf->index());
348 encoded = vf->encode_locally (boost::bind (&Log::dcp_log, _film->log().get(), _1, _2));
349 LOG_TIMING ("finish-local-encode thread=%1 frame=%2", thread_id(), vf->index());
350 } catch (std::exception& e) {
351 /* This is very bad, so don't cope with it, just pass it on */
352 LOG_ERROR (N_("Local encode failed (%1)"), e.what ());
358 _writer->write (encoded.get(), vf->index (), vf->eyes ());
362 LOG_GENERAL (N_("[%1] Encoder thread pushes frame %2 back onto queue after failure"), thread_id(), vf->index());
363 _queue.push_front (vf);
368 if (remote_backoff > 0) {
369 boost::this_thread::sleep (boost::posix_time::seconds (remote_backoff));
372 /* The queue might not be full any more, so notify anything that is waiting on that */
374 _full_condition.notify_all ();
377 catch (boost::thread_interrupted& e) {
378 /* Ignore these and just stop the thread */
379 _full_condition.notify_all ();
384 /* Wake anything waiting on _full_condition so it can see the exception */
385 _full_condition.notify_all ();
389 Encoder::servers_list_changed ()
391 terminate_threads ();
393 /* XXX: could re-use threads */
395 boost::mutex::scoped_lock lm (_threads_mutex);
397 #ifdef BOOST_THREAD_PLATFORM_WIN32
399 info.dwOSVersionInfoSize = sizeof (OSVERSIONINFO);
400 GetVersionEx (&info);
401 bool const windows_xp = (info.dwMajorVersion == 5 && info.dwMinorVersion == 1);
403 LOG_GENERAL_NC (N_("Setting thread affinity for Windows XP"));
407 if (!Config::instance()->only_servers_encode ()) {
408 for (int i = 0; i < Config::instance()->num_local_encoding_threads (); ++i) {
409 boost::thread* t = new boost::thread (boost::bind (&Encoder::encoder_thread, this, optional<EncodeServerDescription> ()));
410 _threads.push_back (t);
411 #ifdef BOOST_THREAD_PLATFORM_WIN32
413 SetThreadAffinityMask (t->native_handle(), 1 << i);
419 BOOST_FOREACH (EncodeServerDescription i, EncodeServerFinder::instance()->servers ()) {
420 LOG_GENERAL (N_("Adding %1 worker threads for remote %2"), i.threads(), i.host_name ());
421 for (int j = 0; j < i.threads(); ++j) {
422 _threads.push_back (new boost::thread (boost::bind (&Encoder::encoder_thread, this, i)));
426 _writer->set_encoder_threads (_threads.size ());