Copy with progress updates when we might copy long files (#1574).
authorCarl Hetherington <cth@carlh.net>
Mon, 21 Oct 2019 15:25:04 +0000 (17:25 +0200)
committerCarl Hetherington <cth@carlh.net>
Mon, 21 Oct 2019 23:38:43 +0000 (23:38 +0000)
Backported from 48b82de5b6e8e07330a2f72dbddd8d9830fe047e in v2.15.x.

src/lib/reel_writer.cc
src/lib/reel_writer.h
src/lib/util.cc
src/lib/util.h
test/test.cc
test/test.h
test/util_test.cc

index c680d3a4e722ed17f0839ea27cbf88c41d471661..a386d19b71f778be0515fc11f92c0823ad950963 100644 (file)
@@ -53,6 +53,7 @@
 using std::list;
 using std::string;
 using std::cout;
 using std::list;
 using std::string;
 using std::cout;
+using std::exception;
 using std::map;
 using boost::shared_ptr;
 using boost::optional;
 using std::map;
 using boost::shared_ptr;
 using boost::optional;
@@ -73,6 +74,7 @@ ReelWriter::ReelWriter (
        , _reel_index (reel_index)
        , _reel_count (reel_count)
        , _content_summary (content_summary)
        , _reel_index (reel_index)
        , _reel_count (reel_count)
        , _content_summary (content_summary)
+       , _job (job)
 {
        /* Create our picture asset in a subdirectory, named according to those
           film's parameters which affect the video output.  We will hard-link
 {
        /* Create our picture asset in a subdirectory, named according to those
           film's parameters which affect the video output.  We will hard-link
@@ -98,9 +100,6 @@ ReelWriter::ReelWriter (
                _film->internal_video_asset_dir() / _film->internal_video_asset_filename(_period)
                );
 
                _film->internal_video_asset_dir() / _film->internal_video_asset_filename(_period)
                );
 
-       if (job) {
-               job->sub (_("Checking existing image data"));
-       }
        _first_nonexistant_frame = check_existing_picture_asset ();
 
        _picture_asset_writer = _picture_asset->start_write (
        _first_nonexistant_frame = check_existing_picture_asset ();
 
        _picture_asset_writer = _picture_asset->start_write (
@@ -178,16 +177,27 @@ ReelWriter::check_existing_picture_asset ()
        DCPOMATIC_ASSERT (_picture_asset->file());
        boost::filesystem::path asset = _picture_asset->file().get();
 
        DCPOMATIC_ASSERT (_picture_asset->file());
        boost::filesystem::path asset = _picture_asset->file().get();
 
+       shared_ptr<Job> job = _job.lock ();
+
        /* If there is an existing asset, break any hard links to it as we are about to change its contents
           (if only by changing the IDs); see #1126.
        */
 
        if (boost::filesystem::exists(asset) && boost::filesystem::hard_link_count(asset) > 1) {
        /* If there is an existing asset, break any hard links to it as we are about to change its contents
           (if only by changing the IDs); see #1126.
        */
 
        if (boost::filesystem::exists(asset) && boost::filesystem::hard_link_count(asset) > 1) {
-               boost::filesystem::copy_file (asset, asset.string() + ".tmp");
+               if (job) {
+                       job->sub (_("Copying old video file"));
+                       copy_in_bits (asset, asset.string() + ".tmp", bind(&Job::set_progress, job.get(), _1, false));
+               } else {
+                       boost::filesystem::copy_file (asset, asset.string() + ".tmp");
+               }
                boost::filesystem::remove (asset);
                boost::filesystem::rename (asset.string() + ".tmp", asset);
        }
 
                boost::filesystem::remove (asset);
                boost::filesystem::rename (asset.string() + ".tmp", asset);
        }
 
+       if (job) {
+               job->sub (_("Checking existing image data"));
+       }
+
        /* Try to open the existing asset */
        FILE* asset_file = fopen_boost (asset, "rb");
        if (!asset_file) {
        /* Try to open the existing asset */
        FILE* asset_file = fopen_boost (asset, "rb");
        if (!asset_file) {
@@ -293,10 +303,21 @@ ReelWriter::finish ()
                boost::filesystem::create_hard_link (video_from, video_to, ec);
                if (ec) {
                        LOG_WARNING_NC ("Hard-link failed; copying instead");
                boost::filesystem::create_hard_link (video_from, video_to, ec);
                if (ec) {
                        LOG_WARNING_NC ("Hard-link failed; copying instead");
-                       boost::filesystem::copy_file (video_from, video_to, ec);
-                       if (ec) {
-                               LOG_ERROR ("Failed to copy video file from %1 to %2 (%3)", video_from.string(), video_to.string(), ec.message ());
-                               throw FileError (ec.message(), video_from);
+                       shared_ptr<Job> job = _job.lock ();
+                       if (job) {
+                               job->sub (_("Copying video file into DCP"));
+                               try {
+                                       copy_in_bits (video_from, video_to, bind(&Job::set_progress, job.get(), _1, false));
+                               } catch (exception& e) {
+                                       LOG_ERROR ("Failed to copy video file from %1 to %2 (%3)", video_from.string(), video_to.string(), e.what());
+                                       throw FileError (e.what(), video_from);
+                               }
+                       } else {
+                               boost::filesystem::copy_file (video_from, video_to, ec);
+                               if (ec) {
+                                       LOG_ERROR ("Failed to copy video file from %1 to %2 (%3)", video_from.string(), video_to.string(), ec.message ());
+                                       throw FileError (ec.message(), video_from);
+                               }
                        }
                }
 
                        }
                }
 
index f13f05dde00bd06f19c515248dd9320d47eb12de..0c9b72184bec8a70768a65239ddf29af43a3a1f5 100644 (file)
@@ -1,5 +1,5 @@
 /*
 /*
-    Copyright (C) 2012-2018 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
 
     This file is part of DCP-o-matic.
 
 
     This file is part of DCP-o-matic.
 
@@ -25,6 +25,7 @@
 #include "dcp_text_track.h"
 #include <dcp/picture_asset_writer.h>
 #include <boost/shared_ptr.hpp>
 #include "dcp_text_track.h"
 #include <dcp/picture_asset_writer.h>
 #include <boost/shared_ptr.hpp>
+#include <boost/weak_ptr.hpp>
 
 class Film;
 class Job;
 
 class Film;
 class Job;
@@ -113,6 +114,7 @@ private:
        /** number of reels in the DCP */
        int _reel_count;
        boost::optional<std::string> _content_summary;
        /** number of reels in the DCP */
        int _reel_count;
        boost::optional<std::string> _content_summary;
+       boost::weak_ptr<Job> _job;
 
        boost::shared_ptr<dcp::PictureAsset> _picture_asset;
        boost::shared_ptr<dcp::PictureAssetWriter> _picture_asset_writer;
 
        boost::shared_ptr<dcp::PictureAsset> _picture_asset;
        boost::shared_ptr<dcp::PictureAssetWriter> _picture_asset_writer;
index 765fc3bf5135b45a7f26e1e72c30ff22452dde9f..a09e6859f834ba867a84544f1f01e8447f37a475 100644 (file)
@@ -44,6 +44,7 @@
 #include "ffmpeg_image_proxy.h"
 #include "image.h"
 #include "text_decoder.h"
 #include "ffmpeg_image_proxy.h"
 #include "image.h"
 #include "text_decoder.h"
+#include "job_manager.h"
 #include <dcp/locale_convert.h>
 #include <dcp/util.h>
 #include <dcp/raw_convert.h>
 #include <dcp/locale_convert.h>
 #include <dcp/util.h>
 #include <dcp/raw_convert.h>
@@ -958,6 +959,113 @@ emit_subtitle_image (ContentTimePeriod period, dcp::SubtitleImage sub, dcp::Size
        decoder->emit_bitmap (period, image, rect);
 }
 
        decoder->emit_bitmap (period, image, rect);
 }
 
+bool
+show_jobs_on_console (bool progress)
+{
+       bool first = true;
+       bool error = false;
+       while (true) {
+
+               dcpomatic_sleep (5);
+
+               list<shared_ptr<Job> > jobs = JobManager::instance()->get();
+
+               if (!first && progress) {
+                       for (size_t i = 0; i < jobs.size(); ++i) {
+                               cout << "\033[1A\033[2K";
+                       }
+                       cout.flush ();
+               }
+
+               first = false;
+
+               BOOST_FOREACH (shared_ptr<Job> i, jobs) {
+                       if (progress) {
+                               cout << i->name();
+                               if (!i->sub_name().empty()) {
+                                       cout << "; " << i->sub_name();
+                               }
+                               cout << ": ";
+
+                               if (i->progress ()) {
+                                       cout << i->status() << "                            \n";
+                               } else {
+                                       cout << ": Running           \n";
+                               }
+                       }
+
+                       if (!progress && i->finished_in_error()) {
+                               /* We won't see this error if we haven't been showing progress,
+                                  so show it now.
+                               */
+                               cout << i->status() << "\n";
+                       }
+
+                       if (i->finished_in_error()) {
+                               error = true;
+                       }
+               }
+
+               if (!JobManager::instance()->work_to_do()) {
+                       break;
+               }
+       }
+
+       return error;
+}
+
+/** XXX: could use mmap? */
+void
+copy_in_bits (boost::filesystem::path from, boost::filesystem::path to, boost::function<void (float)> progress)
+{
+       FILE* f = fopen_boost (from, "rb");
+       if (!f) {
+               throw OpenFileError (from, errno, OpenFileError::READ);
+       }
+       FILE* t = fopen_boost (to, "wb");
+       if (!t) {
+               fclose (f);
+               throw OpenFileError (to, errno, OpenFileError::WRITE);
+       }
+
+       /* on the order of a second's worth of copying */
+       boost::uintmax_t const chunk = 20 * 1024 * 1024;
+
+       uint8_t* buffer = static_cast<uint8_t*> (malloc(chunk));
+       if (!buffer) {
+               throw std::bad_alloc ();
+       }
+
+       boost::uintmax_t const total = boost::filesystem::file_size (from);
+       boost::uintmax_t remaining = total;
+
+       while (remaining) {
+               boost::uintmax_t this_time = min (chunk, remaining);
+               size_t N = fread (buffer, 1, chunk, f);
+               if (N < this_time) {
+                       fclose (f);
+                       fclose (t);
+                       free (buffer);
+                       throw ReadFileError (from, errno);
+               }
+
+               N = fwrite (buffer, 1, this_time, t);
+               if (N < this_time) {
+                       fclose (f);
+                       fclose (t);
+                       free (buffer);
+                       throw WriteFileError (to, errno);
+               }
+
+               progress (1 - float(remaining) / total);
+               remaining -= this_time;
+       }
+
+       fclose (f);
+       fclose (t);
+       free (buffer);
+}
+
 #ifdef DCPOMATIC_VARIANT_SWAROOP
 
 /* Make up a key from the machine UUID */
 #ifdef DCPOMATIC_VARIANT_SWAROOP
 
 /* Make up a key from the machine UUID */
index 5ffdae450f711bc4eba5ea13796e92ebe9796b61..a011da3352ab938eb60c9bc6a112a40f1dda076b 100644 (file)
@@ -1,5 +1,5 @@
 /*
 /*
-    Copyright (C) 2012-2016 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
 
     This file is part of DCP-o-matic.
 
 
     This file is part of DCP-o-matic.
 
@@ -106,6 +106,8 @@ extern void checked_fwrite (void const * ptr, size_t size, FILE* stream, boost::
 extern size_t utf8_strlen (std::string s);
 extern std::string day_of_week_to_string (boost::gregorian::greg_weekday d);
 extern void emit_subtitle_image (ContentTimePeriod period, dcp::SubtitleImage sub, dcp::Size size, boost::shared_ptr<TextDecoder> decoder);
 extern size_t utf8_strlen (std::string s);
 extern std::string day_of_week_to_string (boost::gregorian::greg_weekday d);
 extern void emit_subtitle_image (ContentTimePeriod period, dcp::SubtitleImage sub, dcp::Size size, boost::shared_ptr<TextDecoder> decoder);
+extern bool show_jobs_on_console (bool progress);
+extern void copy_in_bits (boost::filesystem::path from, boost::filesystem::path to, boost::function<void (float)>);
 #ifdef DCPOMATIC_VARIANT_SWAROOP
 extern boost::shared_ptr<dcp::CertificateChain> read_swaroop_chain (boost::filesystem::path path);
 extern void write_swaroop_chain (boost::shared_ptr<const dcp::CertificateChain> chain, boost::filesystem::path output);
 #ifdef DCPOMATIC_VARIANT_SWAROOP
 extern boost::shared_ptr<dcp::CertificateChain> read_swaroop_chain (boost::filesystem::path path);
 extern void write_swaroop_chain (boost::shared_ptr<const dcp::CertificateChain> chain, boost::filesystem::path output);
index c87f4e70b736d9c4e8d561fd8b9c1d0ca22a4fbc..50770a6875ce2e2f05f9685f2319eb2fdc651d40 100644 (file)
@@ -495,3 +495,26 @@ subtitle_file (shared_ptr<Film> film)
        /* Remove warning */
        return boost::filesystem::path("/");
 }
        /* Remove warning */
        return boost::filesystem::path("/");
 }
+
+void
+make_random_file (boost::filesystem::path path, size_t size)
+{
+       size_t const chunk = 128 * 1024;
+       uint8_t* buffer = static_cast<uint8_t*> (malloc(chunk));
+       BOOST_REQUIRE (buffer);
+       FILE* r = fopen("/dev/urandom", "rb");
+       BOOST_REQUIRE (r);
+       FILE* t = fopen_boost(path, "wb");
+       BOOST_REQUIRE (t);
+       while (size) {
+               size_t this_time = min (size, chunk);
+               size_t N = fread (buffer, 1, this_time, r);
+               BOOST_REQUIRE (N == this_time);
+               N = fwrite (buffer, 1, this_time, t);
+               BOOST_REQUIRE (N == this_time);
+               size -= this_time;
+       }
+       fclose (t);
+       fclose (r);
+       free (buffer);
+}
index 4020dc772204d4355038676c1deab86d51d5d571..86b13cde53c97561ef0173d9a0a260d707ed897d 100644 (file)
@@ -42,3 +42,4 @@ extern void write_image (boost::shared_ptr<const Image> image, boost::filesystem
 boost::filesystem::path dcp_file (boost::shared_ptr<const Film> film, std::string prefix);
 void check_one_frame (boost::filesystem::path dcp, int64_t index, boost::filesystem::path ref);
 extern boost::filesystem::path subtitle_file (boost::shared_ptr<Film> film);
 boost::filesystem::path dcp_file (boost::shared_ptr<const Film> film, std::string prefix);
 void check_one_frame (boost::filesystem::path dcp, int64_t index, boost::filesystem::path ref);
 extern boost::filesystem::path subtitle_file (boost::shared_ptr<Film> film);
+extern void make_random_file (boost::filesystem::path path, size_t size);
index 1e13efa82f456959d4cf538c70dd809fbf7c657f..9b2fa989c7b08ebb3657f65c48465a06374a363c 100644 (file)
@@ -1,5 +1,5 @@
 /*
 /*
-    Copyright (C) 2012-2016 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2012-2019 Carl Hetherington <cth@carlh.net>
 
     This file is part of DCP-o-matic.
 
 
     This file is part of DCP-o-matic.
 
 #include "lib/util.h"
 #include "lib/cross.h"
 #include "lib/exceptions.h"
 #include "lib/util.h"
 #include "lib/cross.h"
 #include "lib/exceptions.h"
+#include "test.h"
 #include <dcp/certificate_chain.h>
 #include <boost/test/unit_test.hpp>
 #include <dcp/certificate_chain.h>
 #include <boost/test/unit_test.hpp>
+#include <boost/bind.hpp>
 
 using std::string;
 using std::vector;
 
 using std::string;
 using std::vector;
+using std::list;
 using boost::shared_ptr;
 
 BOOST_AUTO_TEST_CASE (digest_head_tail_test)
 using boost::shared_ptr;
 
 BOOST_AUTO_TEST_CASE (digest_head_tail_test)
@@ -128,3 +131,24 @@ BOOST_AUTO_TEST_CASE (careful_string_filter_test)
        BOOST_CHECK_EQUAL ("hello_world", careful_string_filter("héllo_wörld"));
        BOOST_CHECK_EQUAL ("hello_world_a", careful_string_filter("héllo_wörld_à"));
 }
        BOOST_CHECK_EQUAL ("hello_world", careful_string_filter("héllo_wörld"));
        BOOST_CHECK_EQUAL ("hello_world_a", careful_string_filter("héllo_wörld_à"));
 }
+
+static list<float> progress_values;
+
+static void
+progress (float p)
+{
+       progress_values.push_back (p);
+}
+
+BOOST_AUTO_TEST_CASE (copy_in_bits_test)
+{
+       for (int i = 0; i < 32; ++i) {
+               make_random_file ("build/test/random.dat", rand() % (256 * 1024 * 1024));
+
+               progress_values.clear ();
+               copy_in_bits ("build/test/random.dat", "build/test/random.dat2", boost::bind(&progress, _1));
+               BOOST_CHECK (!progress_values.empty());
+
+               check_file ("build/test/random.dat", "build/test/random.dat2");
+       }
+}