Don't allow the queue to get too big with REPEAT frames otherwise

there's a long delay at the end of the job while they are written.
We must still write FULL frames even if the queue is long (we only
stop doing that if the queue has too many FULL frames i.e. too much
memory consumption).  With this commit we stop writing REPEAT/FAKE
frames when the queue gets long and assume there will always be a
sequence image for writing and hence the main writer thread will reduce
the queue given time.

diff --git a/ChangeLog b/ChangeLog
index 386173b96b6f08819aa909007762b7ea9a6c076f..f740d2e704c66f6c534fa7b7dbf39c7ae16defc5 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2018-06-05  Carl Hetherington  <cth@carlh.net>
+
+       * Don't allow the queue to get too big with repeat frames
+       otherwise there's a long delay at the end of the job
+       while they are written (#1317).
+
 2018-06-04  Carl Hetherington  <cth@carlh.net>
 
        * Updated nl_NL translation from Rob van Nieuwkerk.

diff --git a/src/lib/writer.cc b/src/lib/writer.cc
index 896d59cfcb2d0856e5647887da828a3abd34b616..39d6f61188aa23f3610c77acf0fe4a73452cea1a 100644 (file)
--- a/src/lib/writer.cc
+++ b/src/lib/writer.cc
@@ -75,6 +75,7 @@ Writer::Writer (shared_ptr<const Film> film, weak_ptr<Job> j)
        , _finish (false)
        , _queued_full_in_memory (0)
        , _maximum_frames_in_memory (0)
+       , _maximum_queue_size (0)
        , _full_written (0)
        , _fake_written (0)
        , _repeat_written (0)
@@ -128,7 +129,7 @@ Writer::write (Data encoded, Frame frame, Eyes eyes)
        boost::mutex::scoped_lock lock (_state_mutex);
 
        while (_queued_full_in_memory > _maximum_frames_in_memory) {
-               /* The queue is too big; wait until that is sorted out */
+               /* There are too many full frames in memory; wait until that is sorted out */
                _full_condition.wait (lock);
        }
 
@@ -171,7 +172,7 @@ Writer::repeat (Frame frame, Eyes eyes)
 {
        boost::mutex::scoped_lock lock (_state_mutex);
 
-       while (_queued_full_in_memory > _maximum_frames_in_memory) {
+       while (_queue.size() > _maximum_queue_size) {
                /* The queue is too big; wait until that is sorted out */
                _full_condition.wait (lock);
        }
@@ -199,7 +200,7 @@ Writer::fake_write (Frame frame, Eyes eyes)
 {
        boost::mutex::scoped_lock lock (_state_mutex);
 
-       while (_queued_full_in_memory > _maximum_frames_in_memory) {
+       while (_queue.size() > _maximum_queue_size) {
                /* The queue is too big; wait until that is sorted out */
                _full_condition.wait (lock);
        }
@@ -405,6 +406,7 @@ try
                        }
 
                        lock.lock ();
+                       _full_condition.notify_all ();
                }
 
                while (_queued_full_in_memory > _maximum_frames_in_memory) {
@@ -440,10 +442,8 @@ try
                        lock.lock ();
                        i->encoded.reset ();
                        --_queued_full_in_memory;
+                       _full_condition.notify_all ();
                }
-
-               /* The queue has probably just gone down a bit; notify anything wait()ing on _full_condition */
-               _full_condition.notify_all ();
        }
 }
 catch (...)
@@ -713,6 +713,7 @@ void
 Writer::set_encoder_threads (int threads)
 {
        _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
+       _maximum_queue_size = lrint (threads * 16);
 }
 
 void

diff --git a/src/lib/writer.h b/src/lib/writer.h
index ec7b9880417c9363d13f7d6459b95124afb594f5..0e1c0e02f3e66dc5b5db6d16b9ec7c98bd5274f3 100644 (file)
--- a/src/lib/writer.h
+++ b/src/lib/writer.h
@@ -144,6 +144,7 @@ private:
         *  ordering
         */
        int _maximum_frames_in_memory;
+       unsigned int _maximum_queue_size;
 
        /** number of FULL written frames */
        int _full_written;