From: Carl Hetherington Date: Mon, 21 Oct 2019 15:25:04 +0000 (+0200) Subject: Copy with progress updates when we might copy long files (#1574). X-Git-Tag: v2.14.12~2 X-Git-Url: https://main.carlh.net/gitweb/?p=dcpomatic.git;a=commitdiff_plain;h=c6dba8368b45b6177a971050236d37da0ce4ff4c Copy with progress updates when we might copy long files (#1574). Backported from 48b82de5b6e8e07330a2f72dbddd8d9830fe047e in v2.15.x. --- diff --git a/src/lib/reel_writer.cc b/src/lib/reel_writer.cc index c680d3a4e..a386d19b7 100644 --- a/src/lib/reel_writer.cc +++ b/src/lib/reel_writer.cc @@ -53,6 +53,7 @@ using std::list; using std::string; using std::cout; +using std::exception; using std::map; using boost::shared_ptr; using boost::optional; @@ -73,6 +74,7 @@ ReelWriter::ReelWriter ( , _reel_index (reel_index) , _reel_count (reel_count) , _content_summary (content_summary) + , _job (job) { /* Create our picture asset in a subdirectory, named according to those film's parameters which affect the video output. We will hard-link @@ -98,9 +100,6 @@ ReelWriter::ReelWriter ( _film->internal_video_asset_dir() / _film->internal_video_asset_filename(_period) ); - if (job) { - job->sub (_("Checking existing image data")); - } _first_nonexistant_frame = check_existing_picture_asset (); _picture_asset_writer = _picture_asset->start_write ( @@ -178,16 +177,27 @@ ReelWriter::check_existing_picture_asset () DCPOMATIC_ASSERT (_picture_asset->file()); boost::filesystem::path asset = _picture_asset->file().get(); + shared_ptr job = _job.lock (); + /* If there is an existing asset, break any hard links to it as we are about to change its contents (if only by changing the IDs); see #1126. */ if (boost::filesystem::exists(asset) && boost::filesystem::hard_link_count(asset) > 1) { - boost::filesystem::copy_file (asset, asset.string() + ".tmp"); + if (job) { + job->sub (_("Copying old video file")); + copy_in_bits (asset, asset.string() + ".tmp", bind(&Job::set_progress, job.get(), _1, false)); + } else { + boost::filesystem::copy_file (asset, asset.string() + ".tmp"); + } boost::filesystem::remove (asset); boost::filesystem::rename (asset.string() + ".tmp", asset); } + if (job) { + job->sub (_("Checking existing image data")); + } + /* Try to open the existing asset */ FILE* asset_file = fopen_boost (asset, "rb"); if (!asset_file) { @@ -293,10 +303,21 @@ ReelWriter::finish () boost::filesystem::create_hard_link (video_from, video_to, ec); if (ec) { LOG_WARNING_NC ("Hard-link failed; copying instead"); - boost::filesystem::copy_file (video_from, video_to, ec); - if (ec) { - LOG_ERROR ("Failed to copy video file from %1 to %2 (%3)", video_from.string(), video_to.string(), ec.message ()); - throw FileError (ec.message(), video_from); + shared_ptr job = _job.lock (); + if (job) { + job->sub (_("Copying video file into DCP")); + try { + copy_in_bits (video_from, video_to, bind(&Job::set_progress, job.get(), _1, false)); + } catch (exception& e) { + LOG_ERROR ("Failed to copy video file from %1 to %2 (%3)", video_from.string(), video_to.string(), e.what()); + throw FileError (e.what(), video_from); + } + } else { + boost::filesystem::copy_file (video_from, video_to, ec); + if (ec) { + LOG_ERROR ("Failed to copy video file from %1 to %2 (%3)", video_from.string(), video_to.string(), ec.message ()); + throw FileError (ec.message(), video_from); + } } } diff --git a/src/lib/reel_writer.h b/src/lib/reel_writer.h index f13f05dde..0c9b72184 100644 --- a/src/lib/reel_writer.h +++ b/src/lib/reel_writer.h @@ -1,5 +1,5 @@ /* - Copyright (C) 2012-2018 Carl Hetherington + Copyright (C) 2012-2019 Carl Hetherington This file is part of DCP-o-matic. @@ -25,6 +25,7 @@ #include "dcp_text_track.h" #include #include +#include class Film; class Job; @@ -113,6 +114,7 @@ private: /** number of reels in the DCP */ int _reel_count; boost::optional _content_summary; + boost::weak_ptr _job; boost::shared_ptr _picture_asset; boost::shared_ptr _picture_asset_writer; diff --git a/src/lib/util.cc b/src/lib/util.cc index 765fc3bf5..a09e6859f 100644 --- a/src/lib/util.cc +++ b/src/lib/util.cc @@ -44,6 +44,7 @@ #include "ffmpeg_image_proxy.h" #include "image.h" #include "text_decoder.h" +#include "job_manager.h" #include #include #include @@ -958,6 +959,113 @@ emit_subtitle_image (ContentTimePeriod period, dcp::SubtitleImage sub, dcp::Size decoder->emit_bitmap (period, image, rect); } +bool +show_jobs_on_console (bool progress) +{ + bool first = true; + bool error = false; + while (true) { + + dcpomatic_sleep (5); + + list > jobs = JobManager::instance()->get(); + + if (!first && progress) { + for (size_t i = 0; i < jobs.size(); ++i) { + cout << "\033[1A\033[2K"; + } + cout.flush (); + } + + first = false; + + BOOST_FOREACH (shared_ptr i, jobs) { + if (progress) { + cout << i->name(); + if (!i->sub_name().empty()) { + cout << "; " << i->sub_name(); + } + cout << ": "; + + if (i->progress ()) { + cout << i->status() << " \n"; + } else { + cout << ": Running \n"; + } + } + + if (!progress && i->finished_in_error()) { + /* We won't see this error if we haven't been showing progress, + so show it now. + */ + cout << i->status() << "\n"; + } + + if (i->finished_in_error()) { + error = true; + } + } + + if (!JobManager::instance()->work_to_do()) { + break; + } + } + + return error; +} + +/** XXX: could use mmap? */ +void +copy_in_bits (boost::filesystem::path from, boost::filesystem::path to, boost::function progress) +{ + FILE* f = fopen_boost (from, "rb"); + if (!f) { + throw OpenFileError (from, errno, OpenFileError::READ); + } + FILE* t = fopen_boost (to, "wb"); + if (!t) { + fclose (f); + throw OpenFileError (to, errno, OpenFileError::WRITE); + } + + /* on the order of a second's worth of copying */ + boost::uintmax_t const chunk = 20 * 1024 * 1024; + + uint8_t* buffer = static_cast (malloc(chunk)); + if (!buffer) { + throw std::bad_alloc (); + } + + boost::uintmax_t const total = boost::filesystem::file_size (from); + boost::uintmax_t remaining = total; + + while (remaining) { + boost::uintmax_t this_time = min (chunk, remaining); + size_t N = fread (buffer, 1, chunk, f); + if (N < this_time) { + fclose (f); + fclose (t); + free (buffer); + throw ReadFileError (from, errno); + } + + N = fwrite (buffer, 1, this_time, t); + if (N < this_time) { + fclose (f); + fclose (t); + free (buffer); + throw WriteFileError (to, errno); + } + + progress (1 - float(remaining) / total); + remaining -= this_time; + } + + fclose (f); + fclose (t); + free (buffer); +} + #ifdef DCPOMATIC_VARIANT_SWAROOP /* Make up a key from the machine UUID */ diff --git a/src/lib/util.h b/src/lib/util.h index 5ffdae450..a011da335 100644 --- a/src/lib/util.h +++ b/src/lib/util.h @@ -1,5 +1,5 @@ /* - Copyright (C) 2012-2016 Carl Hetherington + Copyright (C) 2012-2019 Carl Hetherington This file is part of DCP-o-matic. @@ -106,6 +106,8 @@ extern void checked_fwrite (void const * ptr, size_t size, FILE* stream, boost:: extern size_t utf8_strlen (std::string s); extern std::string day_of_week_to_string (boost::gregorian::greg_weekday d); extern void emit_subtitle_image (ContentTimePeriod period, dcp::SubtitleImage sub, dcp::Size size, boost::shared_ptr decoder); +extern bool show_jobs_on_console (bool progress); +extern void copy_in_bits (boost::filesystem::path from, boost::filesystem::path to, boost::function); #ifdef DCPOMATIC_VARIANT_SWAROOP extern boost::shared_ptr read_swaroop_chain (boost::filesystem::path path); extern void write_swaroop_chain (boost::shared_ptr chain, boost::filesystem::path output); diff --git a/test/test.cc b/test/test.cc index c87f4e70b..50770a687 100644 --- a/test/test.cc +++ b/test/test.cc @@ -495,3 +495,26 @@ subtitle_file (shared_ptr film) /* Remove warning */ return boost::filesystem::path("/"); } + +void +make_random_file (boost::filesystem::path path, size_t size) +{ + size_t const chunk = 128 * 1024; + uint8_t* buffer = static_cast (malloc(chunk)); + BOOST_REQUIRE (buffer); + FILE* r = fopen("/dev/urandom", "rb"); + BOOST_REQUIRE (r); + FILE* t = fopen_boost(path, "wb"); + BOOST_REQUIRE (t); + while (size) { + size_t this_time = min (size, chunk); + size_t N = fread (buffer, 1, this_time, r); + BOOST_REQUIRE (N == this_time); + N = fwrite (buffer, 1, this_time, t); + BOOST_REQUIRE (N == this_time); + size -= this_time; + } + fclose (t); + fclose (r); + free (buffer); +} diff --git a/test/test.h b/test/test.h index 4020dc772..86b13cde5 100644 --- a/test/test.h +++ b/test/test.h @@ -42,3 +42,4 @@ extern void write_image (boost::shared_ptr image, boost::filesystem boost::filesystem::path dcp_file (boost::shared_ptr film, std::string prefix); void check_one_frame (boost::filesystem::path dcp, int64_t index, boost::filesystem::path ref); extern boost::filesystem::path subtitle_file (boost::shared_ptr film); +extern void make_random_file (boost::filesystem::path path, size_t size); diff --git a/test/util_test.cc b/test/util_test.cc index 1e13efa82..9b2fa989c 100644 --- a/test/util_test.cc +++ b/test/util_test.cc @@ -1,5 +1,5 @@ /* - Copyright (C) 2012-2016 Carl Hetherington + Copyright (C) 2012-2019 Carl Hetherington This file is part of DCP-o-matic. @@ -26,11 +26,14 @@ #include "lib/util.h" #include "lib/cross.h" #include "lib/exceptions.h" +#include "test.h" #include #include +#include using std::string; using std::vector; +using std::list; using boost::shared_ptr; BOOST_AUTO_TEST_CASE (digest_head_tail_test) @@ -128,3 +131,24 @@ BOOST_AUTO_TEST_CASE (careful_string_filter_test) BOOST_CHECK_EQUAL ("hello_world", careful_string_filter("héllo_wörld")); BOOST_CHECK_EQUAL ("hello_world_a", careful_string_filter("héllo_wörld_à")); } + +static list progress_values; + +static void +progress (float p) +{ + progress_values.push_back (p); +} + +BOOST_AUTO_TEST_CASE (copy_in_bits_test) +{ + for (int i = 0; i < 32; ++i) { + make_random_file ("build/test/random.dat", rand() % (256 * 1024 * 1024)); + + progress_values.clear (); + copy_in_bits ("build/test/random.dat", "build/test/random.dat2", boost::bind(&progress, _1)); + BOOST_CHECK (!progress_values.empty()); + + check_file ("build/test/random.dat", "build/test/random.dat2"); + } +}