src/tools/dcpomatic_dist_writer.cc

   1 /*
   2     Copyright (C) 2019-2020 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "lib/dist_writer_messages.h"
  22 #include "lib/compose.hpp"
  23 #include "lib/exceptions.h"
  24 #include "lib/cross.h"
  25 #include "lib/digester.h"
  26 #include "lib/file_log.h"
  27 #include "lib/dcpomatic_log.h"
  28 extern "C" {
  29 #include <lwext4/ext4_mbr.h>
  30 #include <lwext4/ext4_fs.h>
  31 #include <lwext4/ext4_mkfs.h>
  32 #include <lwext4/ext4_errno.h>
  33 #include <lwext4/ext4_debug.h>
  34 #include <lwext4/ext4.h>
  35 }
  36
  37 #ifdef DCPOMATIC_POSIX
  38 #include <sys/ioctl.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #endif
  42
  43 #ifdef DCPOMATIC_OSX
  44 #undef nil
  45 #endif
  46
  47 #ifdef DCPOMATIC_LINUX
  48 #include <linux/fs.h>
  49 #include <polkit/polkit.h>
  50 extern "C" {
  51 #include <lwext4/file_dev.h>
  52 }
  53 #include <poll.h>
  54 #endif
  55
  56 #ifdef DCPOMATIC_WINDOWS
  57 extern "C" {
  58 #include <lwext4/file_windows.h>
  59 }
  60 #endif
  61
  62 #include <glibmm.h>
  63 #include <unistd.h>
  64 #include <sys/types.h>
  65 #include <boost/filesystem.hpp>
  66 #include <iostream>
  67
  68 using std::cout;
  69 using std::cin;
  70 using std::min;
  71 using std::string;
  72
  73 #ifdef DCPOMATIC_LINUX
  74 static PolkitAuthority* polkit_authority = 0;
  75 #endif
  76 static boost::filesystem::path dcp_path;
  77 static std::string device;
  78 static uint64_t const block_size = 4096;
  79
  80 static
  81 void
  82 count (boost::filesystem::path dir, uint64_t& total_bytes)
  83 {
  84         using namespace boost::filesystem;
  85         for (directory_iterator i = directory_iterator(dir); i != directory_iterator(); ++i) {
  86                 if (is_directory(*i)) {
  87                         count (*i, total_bytes);
  88                 } else {
  89                         total_bytes += file_size (*i);
  90                 }
  91         }
  92 }
  93
  94 static
  95 string
  96 write (boost::filesystem::path from, boost::filesystem::path to, uint64_t& total_remaining, uint64_t total)
  97 {
  98         ext4_file out;
  99         int r = ext4_fopen(&out, to.string().c_str(), "wb");
 100         if (r != EOK) {
 101                 throw CopyError (String::compose("Failed to open file %1", to.string()), r);
 102         }
 103
 104         FILE* in = fopen_boost (from, "rb");
 105         if (!in) {
 106                 ext4_fclose (&out);
 107                 throw CopyError (String::compose("Failed to open file %1", from.string()), 0);
 108         }
 109
 110         uint8_t* buffer = new uint8_t[block_size];
 111         Digester digester;
 112
 113         uint64_t remaining = file_size (from);
 114         while (remaining > 0) {
 115                 uint64_t const this_time = min(remaining, block_size);
 116                 size_t read = fread (buffer, 1, this_time, in);
 117                 if (read != this_time) {
 118                         fclose (in);
 119                         ext4_fclose (&out);
 120                         delete[] buffer;
 121                         throw CopyError (String::compose("Short read; expected %1 but read %2", this_time, read), 0);
 122                 }
 123
 124                 digester.add (buffer, this_time);
 125
 126                 size_t written;
 127                 r = ext4_fwrite (&out, buffer, this_time, &written);
 128                 if (r != EOK) {
 129                         fclose (in);
 130                         ext4_fclose (&out);
 131                         delete[] buffer;
 132                         throw CopyError ("Write failed", r);
 133                 }
 134                 if (written != this_time) {
 135                         fclose (in);
 136                         ext4_fclose (&out);
 137                         delete[] buffer;
 138                         throw CopyError (String::compose("Short write; expected %1 but wrote %2", this_time, written), 0);
 139                 }
 140                 remaining -= this_time;
 141                 total_remaining -= this_time;
 142                 cout << DIST_WRITER_PROGRESS "\n" << (1 - float(total_remaining) / total) << "\n";
 143                 cout.flush ();
 144         }
 145
 146         fclose (in);
 147         ext4_fclose (&out);
 148         delete[] buffer;
 149
 150         return digester.get ();
 151 }
 152
 153 static
 154 string
 155 read (boost::filesystem::path from, boost::filesystem::path to, uint64_t& total_remaining, uint64_t total)
 156 {
 157         ext4_file in;
 158         int r = ext4_fopen(&in, to.string().c_str(), "rb");
 159         if (r != EOK) {
 160                 throw VerifyError (String::compose("Failed to open file %1", to.string()), r);
 161         }
 162
 163         uint8_t* buffer = new uint8_t[block_size];
 164         Digester digester;
 165
 166         uint64_t remaining = file_size (from);
 167         while (remaining > 0) {
 168                 uint64_t const this_time = min(remaining, block_size);
 169                 size_t read;
 170                 r = ext4_fread (&in, buffer, this_time, &read);
 171                 if (read != this_time) {
 172                         ext4_fclose (&in);
 173                         delete[] buffer;
 174                         throw VerifyError (String::compose("Short read; expected %1 but read %2", this_time, read), 0);
 175                 }
 176
 177                 digester.add (buffer, this_time);
 178                 remaining -= this_time;
 179                 total_remaining -= this_time;
 180                 cout << DIST_WRITER_PROGRESS "\n" << (1 - float(total_remaining) / total) << "\n";
 181                 cout.flush ();
 182         }
 183
 184         ext4_fclose (&in);
 185         delete[] buffer;
 186
 187         return digester.get ();
 188 }
 189
 190
 191 /** @param from File to copy from.
 192  *  @param to Directory to copy to.
 193  */
 194 static
 195 void
 196 copy (boost::filesystem::path from, boost::filesystem::path to, uint64_t& total_remaining, uint64_t total)
 197 {
 198         using namespace boost::filesystem;
 199
 200         /* XXX: this is a hack.  We are going to "treat" every byte twice; write it, and then verify it.  Double the
 201          * bytes totals so that progress works itself out (assuming write is the same speed as read).
 202          */
 203         total_remaining *= 2;
 204         total *= 2;
 205
 206         path const cr = to / from.filename();
 207
 208         if (is_directory(from)) {
 209                 int r = ext4_dir_mk (cr.string().c_str());
 210                 if (r != EOK) {
 211                         throw CopyError (String::compose("Failed to create directory %1", cr.string()), r);
 212                 }
 213
 214                 for (directory_iterator i = directory_iterator(from); i != directory_iterator(); ++i) {
 215                         copy (i->path(), cr, total_remaining, total);
 216                 }
 217         } else {
 218                 string const write_digest = write (from, cr, total_remaining, total);
 219                 string const read_digest = read (from, cr, total_remaining, total);
 220                 if (write_digest != read_digest) {
 221                         throw VerifyError ("Hash of written data is incorrect", 0);
 222                 }
 223         }
 224 }
 225
 226 static
 227 void
 228 write ()
 229 try
 230 {
 231 //      ext4_dmask_set (DEBUG_ALL);
 232
 233         cout << "U\n";
 234
 235         /* We rely on static initialization for these */
 236         static struct ext4_fs fs;
 237         static struct ext4_mkfs_info info;
 238         info.block_size = 1024;
 239         info.inode_size = 128;
 240         info.journal = false;
 241
 242 #ifdef WIN32
 243         file_windows_name_set(device.c_str());
 244         struct ext4_blockdev* bd = file_windows_dev_get();
 245 #else
 246         file_dev_name_set (device.c_str());
 247         struct ext4_blockdev* bd = file_dev_get ();
 248 #endif
 249
 250         if (!bd) {
 251                 throw CopyError ("Failed to open drive", 0);
 252         }
 253
 254         struct ext4_mbr_parts parts;
 255         parts.division[0] = 100;
 256         parts.division[1] = 0;
 257         parts.division[2] = 0;
 258         parts.division[3] = 0;
 259
 260 #ifdef DCPOMATIC_LINUX
 261         PrivilegeEscalator e;
 262 #endif
 263
 264         /* XXX: not sure if disk_id matters */
 265         int r = ext4_mbr_write (bd, &parts, 0);
 266
 267         if (r) {
 268                 throw CopyError ("Failed to write MBR", r);
 269         }
 270
 271 #ifdef DCPOMATIC_WINDOWS
 272         struct ext4_mbr_bdevs bdevs;
 273         r = ext4_mbr_scan (bd, &bdevs);
 274         if (r != EOK) {
 275                 throw CopyError ("Failed to read MBR", r);
 276         }
 277
 278         file_windows_partition_set (bdevs.partitions[0].part_offset, bdevs.partitions[0].part_size);
 279 #endif
 280
 281 #ifdef DCPOMATIC_LINUX
 282         /* Re-read the partition table */
 283         int fd = open(device.c_str(), O_RDONLY);
 284         ioctl(fd, BLKRRPART, NULL);
 285         close(fd);
 286 #endif
 287
 288 #ifdef DCPOMATIC_POSIX
 289         string partition = device;
 290         /* XXX: don't know if this logic is sensible */
 291         if (partition.size() > 0 && isdigit(partition[partition.length() - 1])) {
 292                 partition += "p1";
 293         } else {
 294                 partition += "1";
 295         }
 296         file_dev_name_set (partition.c_str());
 297         bd = file_dev_get ();
 298 #endif
 299
 300         if (!bd) {
 301                 throw CopyError ("Failed to open partition", 0);
 302         }
 303
 304         r = ext4_mkfs(&fs, bd, &info, F_SET_EXT4);
 305         if (r != EOK) {
 306                 throw CopyError ("Failed to make filesystem", r);
 307         }
 308
 309         r = ext4_device_register(bd, "ext4_fs");
 310         if (r != EOK) {
 311                 throw CopyError ("Failed to register device", r);
 312         }
 313
 314         r = ext4_mount("ext4_fs", "/mp/", false);
 315         if (r != EOK) {
 316                 throw CopyError ("Failed to mount device", r);
 317         }
 318
 319         uint64_t total_bytes = 0;
 320         count (dcp_path, total_bytes);
 321
 322         copy (dcp_path, "/mp", total_bytes, total_bytes);
 323
 324         r = ext4_umount("/mp/");
 325         if (r != EOK) {
 326                 throw CopyError ("Failed to unmount device", r);
 327         }
 328
 329         cout << DIST_WRITER_OK "\n";
 330         cout.flush ();
 331 } catch (CopyError& e) {
 332         cout << DIST_WRITER_ERROR "\n" << e.message() << "\n" << e.number() << "\n";
 333         cout.flush ();
 334 } catch (VerifyError& e) {
 335         cout << DIST_WRITER_ERROR "\n" << e.message() << "\n" << e.number() << "\n";
 336         cout.flush ();
 337 }
 338
 339 #ifdef DCPOMATIC_LINUX
 340 static
 341 void
 342 polkit_callback (GObject *, GAsyncResult* res, gpointer)
 343 {
 344         PolkitAuthorizationResult* result = polkit_authority_check_authorization_finish (polkit_authority, res, 0);
 345         if (result && polkit_authorization_result_get_is_authorized(result)) {
 346                 write ();
 347         }
 348         if (result) {
 349                 g_object_unref (result);
 350         }
 351 }
 352 #endif
 353
 354 bool
 355 idle ()
 356 {
 357 #ifdef DCPOMATIC_POSIX
 358         struct pollfd input[1] = { { fd: 0, events: POLLIN, revents: 0 } };
 359         int const r = poll (input, 1, 0);
 360         if (r > 0 && (input[0].revents & POLLIN)) {
 361 #else
 362         DWORD num;
 363         LOG_DIST ("now handle in is %1", reinterpret_cast<uint64_t>(GetStdHandle(STD_INPUT_HANDLE)));
 364         if (!GetNumberOfConsoleInputEvents(GetStdHandle(STD_INPUT_HANDLE), &num)) {
 365                 LOG_DIST ("Could not check console: %1", GetLastError());
 366         }
 367
 368         LOG_DIST ("%1 console events", num);
 369         if (num) {
 370 #endif
 371                 string s;
 372                 getline (cin, s);
 373                 if (s.empty()) {
 374                         return true;
 375                 }
 376                 dcp_path = s;
 377                 getline (cin, s);
 378                 device = "/dev/" + s;
 379
 380                 LOG_DIST ("Here we go writing %1 to %2", dcp_path, device);
 381
 382 #ifdef DCPOMATIC_LINUX
 383                 polkit_authority = polkit_authority_get_sync (0, 0);
 384                 PolkitSubject* subject = polkit_unix_process_new (getppid());
 385                 polkit_authority_check_authorization (
 386                         polkit_authority, subject, "com.dcpomatic.write-drive", 0, POLKIT_CHECK_AUTHORIZATION_FLAGS_ALLOW_USER_INTERACTION, 0, polkit_callback, 0
 387                         );
 388 #else
 389                 write ();
 390 #endif
 391         }
 392         return true;
 393 }
 394
 395 int
 396 main ()
 397 {
 398         /* XXX: this is a hack, but I expect we'll need logs and I'm not sure if there's
 399          * a better place to put them.
 400          */
 401         dcpomatic_log.reset(new FileLog(config_path() / "dist_writer.log"));
 402         dcpomatic_log->set_types (dcpomatic_log->types() | LogEntry::TYPE_DIST);
 403         LOG_DIST_NC("dcpomatic_dist_writer started");
 404
 405 #ifdef DCPOMATIC_WINDOWS
 406         FreeConsole ();
 407         AllocConsole ();
 408
 409         HANDLE handle_out = GetStdHandle(STD_OUTPUT_HANDLE);
 410         int hCrt = _open_osfhandle((intptr_t) handle_out, _O_TEXT);
 411         FILE* hf_out = _fdopen(hCrt, "w");
 412         setvbuf(hf_out, NULL, _IONBF, 1);
 413         *stdout = *hf_out;
 414
 415         HANDLE handle_in = GetStdHandle(STD_INPUT_HANDLE);
 416         LOG_DIST ("handle_in is %1", reinterpret_cast<uint64_t>(handle_in));
 417         hCrt = _open_osfhandle((intptr_t) handle_in, _O_TEXT);
 418         FILE* hf_in = _fdopen(hCrt, "r");
 419         setvbuf(hf_in, NULL, _IONBF, 128);
 420         *stdin = *hf_in;
 421 #endif
 422
 423         Glib::RefPtr<Glib::MainLoop> ml = Glib::MainLoop::create ();
 424         Glib::signal_timeout().connect(sigc::ptr_fun(&idle), 500);
 425         ml->run ();
 426 }