2 * Copyright (C) 2016-2017 Robin Gareus <robin@gareus.org>
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License along
15 * with this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
31 #include "pbd/gstdio_compat.h"
35 #include <archive_entry.h>
36 #include <curl/curl.h>
38 #include "pbd/failed_constructor.h"
39 #include "pbd/file_archive.h"
40 #include "pbd/file_utils.h"
45 write_callback (void* buffer, size_t size, size_t nmemb, void* d)
47 FileArchive::MemPipe* p = (FileArchive::MemPipe*)d;
48 size_t realsize = size * nmemb;
51 p->data = (uint8_t*) realloc ((void*) p->data, p->size + realsize);
52 memcpy (&p->data[p->size], buffer, realsize);
62 FileArchive::Request* r = (FileArchive::Request*) arg;
65 curl = curl_easy_init ();
66 curl_easy_setopt (curl, CURLOPT_URL, r->url);
67 curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L);
71 curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
72 curl_easy_setopt(curl, CURLOPT_HEADER, 0L);
73 curl_easy_perform (curl);
74 curl_easy_getinfo (curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &r->mp.length);
77 curl_easy_setopt(curl, CURLOPT_NOBODY, 0L);
78 curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_callback);
79 curl_easy_setopt (curl, CURLOPT_WRITEDATA, (void*) &r->mp);
80 curl_easy_perform (curl);
81 curl_easy_cleanup (curl);
92 ar_read (struct archive* a, void* d, const void** buff)
94 FileArchive::MemPipe* p = (FileArchive::MemPipe*)d;
98 while (p->size == 0) {
106 rv = p->size > 8192 ? 8192 : p->size;
107 memcpy (p->buf, p->data, rv);
109 memmove (p->data, &p->data[rv], p->size - rv);
115 p->progress->progress (p->processed, p->length);
122 ar_copy_data (struct archive *ar, struct archive *aw)
129 r = archive_read_data_block (ar, &buff, &size, &offset);
130 if (r == ARCHIVE_EOF) {
133 if (r != ARCHIVE_OK) {
136 r = archive_write_data_block (aw, buff, size, offset);
137 if (r != ARCHIVE_OK) {
138 fprintf (stderr, "Extract/Write Archive: %s", archive_error_string(aw));
144 static struct archive*
148 a = archive_read_new ();
149 archive_read_support_filter_all (a);
150 archive_read_support_format_all (a);
154 FileArchive::FileArchive (const std::string& url)
158 fprintf (stderr, "Invalid Archive URL/filename\n");
159 throw failed_constructor ();
162 if (_req.is_remote ()) {
163 _req.mp.progress = this;
165 _req.mp.progress = 0;
170 FileArchive::inflate (const std::string& destdir)
173 std::string pwd (Glib::get_current_dir ());
175 if (g_chdir (destdir.c_str ())) {
176 fprintf (stderr, "Archive: cannot chdir to '%s'\n", destdir.c_str ());
180 if (_req.is_remote ()) {
183 rv = extract_file ();
186 g_chdir (pwd.c_str());
190 std::vector<std::string>
191 FileArchive::contents ()
193 if (_req.is_remote ()) {
194 return contents_url ();
196 return contents_file ();
200 std::vector<std::string>
201 FileArchive::contents_file ()
203 struct archive* a = setup_archive ();
205 if (!g_stat (_req.url, &statbuf)) {
206 _req.mp.length = statbuf.st_size;
210 if (ARCHIVE_OK != archive_read_open_filename (a, _req.url, 8192)) {
211 fprintf (stderr, "Error opening archive: %s\n", archive_error_string(a));
212 return std::vector<std::string> ();
214 return get_contents (a);
217 std::vector<std::string>
218 FileArchive::contents_url ()
221 pthread_create (&_tid, NULL, get_url, (void*)&_req);
223 struct archive* a = setup_archive ();
224 archive_read_open (a, (void*)&_req.mp, NULL, ar_read, NULL);
225 std::vector<std::string> rv (get_contents (a));
227 pthread_join (_tid, NULL);
232 FileArchive::extract_file ()
234 struct archive* a = setup_archive ();
236 if (!g_stat (_req.url, &statbuf)) {
237 _req.mp.length = statbuf.st_size;
241 if (ARCHIVE_OK != archive_read_open_filename (a, _req.url, 8192)) {
242 fprintf (stderr, "Error opening archive: %s\n", archive_error_string(a));
245 return do_extract (a);
249 FileArchive::extract_url ()
252 pthread_create (&_tid, NULL, get_url, (void*)&_req);
254 struct archive* a = setup_archive ();
255 archive_read_open (a, (void*)&_req.mp, NULL, ar_read, NULL);
256 int rv = do_extract (a);
258 pthread_join (_tid, NULL);
262 std::vector<std::string>
263 FileArchive::get_contents (struct archive* a)
265 std::vector<std::string> rv;
266 struct archive_entry* entry;
268 int r = archive_read_next_header (a, &entry);
269 if (!_req.mp.progress) {
270 // file i/o -- not URL
271 const uint64_t read = archive_filter_bytes (a, -1);
272 progress (read, _req.mp.length);
274 if (r == ARCHIVE_EOF) {
277 if (r != ARCHIVE_OK) {
278 fprintf (stderr, "Error reading archive: %s\n", archive_error_string(a));
281 rv.push_back (archive_entry_pathname (entry));
284 archive_read_close (a);
285 archive_read_free (a);
290 FileArchive::do_extract (struct archive* a)
292 int flags = ARCHIVE_EXTRACT_TIME;
295 struct archive_entry* entry;
298 ext = archive_write_disk_new();
299 archive_write_disk_set_options(ext, flags);
302 int r = archive_read_next_header (a, &entry);
303 if (!_req.mp.progress) {
304 // file i/o -- not URL
305 const uint64_t read = archive_filter_bytes (a, -1);
306 progress (read, _req.mp.length);
309 if (r == ARCHIVE_EOF) {
312 if (r != ARCHIVE_OK) {
313 fprintf (stderr, "Error reading archive: %s\n", archive_error_string(a));
317 #if 0 // hacky alternative to chdir
318 const std::string full_path = Glib::build_filename (destdir, archive_entry_pathname (entry));
319 archive_entry_set_pathname (entry, full_path.c_str());
322 r = archive_write_header(ext, entry);
323 if (r != ARCHIVE_OK) {
324 fprintf (stderr, "Extracting archive: %s\n", archive_error_string(ext));
326 ar_copy_data (a, ext);
327 r = archive_write_finish_entry (ext);
328 if (r != ARCHIVE_OK) {
329 fprintf (stderr, "Extracting archive: %s\n", archive_error_string(ext));
336 archive_read_close (a);
337 archive_read_free (a);
338 archive_write_close(ext);
339 archive_write_free(ext);
345 FileArchive::create (const std::string& srcdir, CompressionLevel compression_level)
347 if (_req.is_remote ()) {
351 std::string parent = Glib::path_get_dirname (srcdir);
352 size_t p_len = parent.size () + 1;
354 Searchpath sp (srcdir);
355 std::vector<std::string> files;
356 find_files_matching_pattern (files, sp, "*");
358 std::map<std::string, std::string> filemap;
360 for (std::vector<std::string>::const_iterator f = files.begin (); f != files.end (); ++f) {
361 assert (f->size () > p_len);
362 filemap[*f] = f->substr (p_len);
365 return create (filemap, compression_level);
369 FileArchive::create (const std::map<std::string, std::string>& filemap, CompressionLevel compression_level)
372 struct archive_entry *entry;
374 size_t read_bytes = 0;
375 size_t total_bytes = 0;
377 for (std::map<std::string, std::string>::const_iterator f = filemap.begin (); f != filemap.end (); ++f) {
379 if (g_stat (f->first.c_str(), &statbuf)) {
382 total_bytes += statbuf.st_size;
385 if (total_bytes == 0) {
389 progress (0, total_bytes);
391 a = archive_write_new ();
392 archive_write_set_format_pax_restricted (a);
394 if (compression_level != CompressNone) {
395 archive_write_add_filter_lzma (a);
397 sprintf (buf, "lzma:compression-level=%u,lzma:threads=0", (uint32_t) compression_level);
398 archive_write_set_options (a, buf);
401 archive_write_open_filename (a, _req.url);
402 entry = archive_entry_new ();
405 const int64_t archive_start_time = g_get_monotonic_time();
408 for (std::map<std::string, std::string>::const_iterator f = filemap.begin (); f != filemap.end (); ++f) {
410 const char* filepath = f->first.c_str ();
411 const char* filename = f->second.c_str ();
414 if (g_stat (filepath, &statbuf)) {
418 archive_entry_clear (entry);
420 #ifdef PLATFORM_WINDOWS
421 archive_entry_set_size (entry, statbuf.st_size);
422 archive_entry_set_atime (entry, statbuf.st_atime, 0);
423 archive_entry_set_ctime (entry, statbuf.st_ctime, 0);
424 archive_entry_set_mtime (entry, statbuf.st_mtime, 0);
426 archive_entry_copy_stat (entry, &statbuf);
429 archive_entry_set_pathname (entry, filename);
430 archive_entry_set_filetype (entry, AE_IFREG);
431 archive_entry_set_perm (entry, 0644);
433 archive_write_header (a, entry);
435 int fd = g_open (filepath, O_RDONLY, 0444);
438 ssize_t len = read (fd, buf, sizeof (buf));
441 archive_write_data (a, buf, len);
442 progress (read_bytes, total_bytes);
443 len = read (fd, buf, sizeof (buf));
448 archive_entry_free (entry);
449 archive_write_close (a);
450 archive_write_free (a);
453 const int64_t elapsed_time_us = g_get_monotonic_time() - archive_start_time;
454 std::cerr << "archived in " << std::fixed << std::setprecision (2) << elapsed_time_us / 1000000. << " sec\n";