From 1d5462ef8d6a32b964f0335e3dd68aac31075d14 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Fri, 17 Jul 2015 15:37:23 +0100 Subject: [PATCH] Add FTP uploader using curl. --- src/lib/config.cc | 4 ++ src/lib/config.h | 9 ++++ src/lib/curl_uploader.cc | 109 +++++++++++++++++++++++++++++++++++++++ src/lib/curl_uploader.h | 41 +++++++++++++++ src/lib/scp_uploader.cc | 4 -- src/lib/types.h | 5 ++ src/lib/upload_job.cc | 24 +++++---- src/lib/uploader.cc | 10 ++-- src/lib/uploader.h | 8 ++- src/lib/wscript | 1 + src/wx/config_dialog.cc | 15 ++++++ 11 files changed, 212 insertions(+), 18 deletions(-) create mode 100644 src/lib/curl_uploader.cc create mode 100644 src/lib/curl_uploader.h diff --git a/src/lib/config.cc b/src/lib/config.cc index b5e87267a..6eecc8574 100644 --- a/src/lib/config.cc +++ b/src/lib/config.cc @@ -21,6 +21,7 @@ #include "server.h" #include "filter.h" #include "ratio.h" +#include "types.h" #include "dcp_content_type.h" #include "cinema_sound_processor.h" #include "colour_conversion.h" @@ -67,6 +68,7 @@ Config::set_defaults () _num_local_encoding_threads = max (2U, boost::thread::hardware_concurrency ()); _server_port_base = 6192; _use_any_servers = true; + _tms_protocol = PROTOCOL_SCP; _tms_path = "."; _cinema_sound_processor = CinemaSoundProcessor::from_id (N_("dolby_cp750")); _allow_any_dcp_frame_rate = false; @@ -140,6 +142,7 @@ Config::read () } } + _tms_protocol = static_cast (f.optional_number_child ("TMSProtocol").get_value_or (static_cast (PROTOCOL_SCP))); _tms_ip = f.string_child ("TMSIP"); _tms_path = f.string_child ("TMSPath"); _tms_user = f.string_child ("TMSUser"); @@ -315,6 +318,7 @@ Config::write () const root->add_child("Server")->add_child_text (*i); } + root->add_child("TMSProtocol")->add_child_text (raw_convert (_tms_protocol)); root->add_child("TMSIP")->add_child_text (_tms_ip); root->add_child("TMSPath")->add_child_text (_tms_path); root->add_child("TMSUser")->add_child_text (_tms_user); diff --git a/src/lib/config.h b/src/lib/config.h index 312118a55..b008fe22f 100644 --- a/src/lib/config.h +++ b/src/lib/config.h @@ -90,6 +90,10 @@ public: return _servers; } + Protocol tms_protocol () const { + return _tms_protocol; + } + /** @return The IP address of a TMS that we can copy DCPs to */ std::string tms_ip () const { return _tms_ip; @@ -247,6 +251,10 @@ public: maybe_set (_server_port_base, p); } + void set_tms_protocol (Protocol p) { + maybe_set (_tms_protocol, p); + } + /** @param i IP address of a TMS that we can copy DCPs to */ void set_tms_ip (std::string i) { maybe_set (_tms_ip, i); @@ -447,6 +455,7 @@ private: bool _use_any_servers; /** J2K encoding servers that should definitely be used */ std::vector _servers; + Protocol _tms_protocol; /** The IP address of a TMS that we can copy DCPs to */ std::string _tms_ip; /** The path on a TMS that we should write DCPs to */ diff --git a/src/lib/curl_uploader.cc b/src/lib/curl_uploader.cc new file mode 100644 index 000000000..b88898544 --- /dev/null +++ b/src/lib/curl_uploader.cc @@ -0,0 +1,109 @@ +/* + Copyright (C) 2015 Carl Hetherington + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +*/ + +#include "curl_uploader.h" +#include "exceptions.h" +#include "config.h" +#include "cross.h" +#include "compose.hpp" + +#include "i18n.h" + +using std::string; +using std::cout; +using boost::function; + +static size_t +read_callback (void* ptr, size_t size, size_t nmemb, void* object) +{ + CurlUploader* u = reinterpret_cast (object); + return u->read_callback (ptr, size, nmemb); +} + +CurlUploader::CurlUploader (function set_status, function set_progress) + : Uploader (set_status, set_progress) + , _file (0) + , _transferred (0) + , _total_size (0) +{ + _curl = curl_easy_init (); + if (!_curl) { + throw NetworkError (_("Could not start transfer")); + } + + curl_easy_setopt (_curl, CURLOPT_READFUNCTION, ::read_callback); + curl_easy_setopt (_curl, CURLOPT_READDATA, this); + curl_easy_setopt (_curl, CURLOPT_UPLOAD, 1L); + curl_easy_setopt (_curl, CURLOPT_FTP_CREATE_MISSING_DIRS, 1L); + curl_easy_setopt (_curl, CURLOPT_READDATA, this); + curl_easy_setopt (_curl, CURLOPT_USERNAME, Config::instance()->tms_user().c_str ()); + curl_easy_setopt (_curl, CURLOPT_PASSWORD, Config::instance()->tms_password().c_str ()); +} + +CurlUploader::~CurlUploader () +{ + if (_file) { + fclose (_file); + } + curl_easy_cleanup (_curl); +} + +void +CurlUploader::create_directory (boost::filesystem::path) +{ + /* this is done by libcurl */ +} + +void +CurlUploader::upload_file (boost::filesystem::path from, boost::filesystem::path to, boost::uintmax_t& transferred, boost::uintmax_t total_size) +{ + curl_easy_setopt ( + _curl, CURLOPT_URL, + String::compose ("ftp://%1/%2/%3", Config::instance()->tms_ip(), Config::instance()->tms_path(), to.string ()).c_str () + ); + + cout << String::compose ("ftp://%1/%2/%3", Config::instance()->tms_ip(), Config::instance()->tms_path(), to.string ()) << "\n"; + + _file = fopen_boost (from, "rb"); + if (!_file) { + throw NetworkError (String::compose (_("Could not open %1 to send"), from)); + } + _transferred = &transferred; + _total_size = total_size; + + CURLcode const r = curl_easy_perform (_curl); + if (r != CURLE_OK) { + throw NetworkError (String::compose (_("Could not write to remote file (%1)"), curl_easy_strerror (r))); + } + + fclose (_file); +} + +size_t +CurlUploader::read_callback (void* ptr, size_t size, size_t nmemb) +{ + size_t const r = fread (ptr, size, nmemb, _file); + *_transferred += size * nmemb; + + if (_total_size > 0) { + _set_progress ((double) *_transferred / _total_size); + } + + return r; +} diff --git a/src/lib/curl_uploader.h b/src/lib/curl_uploader.h new file mode 100644 index 000000000..d83c71eec --- /dev/null +++ b/src/lib/curl_uploader.h @@ -0,0 +1,41 @@ +/* + Copyright (C) 2015 Carl Hetherington + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +*/ + +#include "uploader.h" +#include + +class CurlUploader : public Uploader +{ +public: + CurlUploader (boost::function set_status, boost::function set_progress); + ~CurlUploader (); + + size_t read_callback (void* ptr, size_t size, size_t nmemb); + +protected: + virtual void create_directory (boost::filesystem::path directory); + virtual void upload_file (boost::filesystem::path from, boost::filesystem::path to, boost::uintmax_t& transferred, boost::uintmax_t total_size); + +private: + CURL* _curl; + + FILE* _file; + boost::uintmax_t* _transferred; + boost::uintmax_t _total_size; +}; diff --git a/src/lib/scp_uploader.cc b/src/lib/scp_uploader.cc index 7fb716813..5e2d2de13 100644 --- a/src/lib/scp_uploader.cc +++ b/src/lib/scp_uploader.cc @@ -39,8 +39,6 @@ SCPUploader::SCPUploader (function set_status, functiontms_ip().c_str ()); ssh_options_set (_session, SSH_OPTIONS_USER, Config::instance()->tms_user().c_str ()); int const port = 22; @@ -91,8 +89,6 @@ SCPUploader::create_directory (boost::filesystem::path directory) void SCPUploader::upload_file (boost::filesystem::path from, boost::filesystem::path to, boost::uintmax_t& transferred, boost::uintmax_t total_size) { - _set_status (String::compose (_("copying %1"), from.leaf ())); - boost::uintmax_t to_do = boost::filesystem::file_size (from); ssh_scp_push_file (_scp, to.string().c_str(), to_do, S_IRUSR | S_IWUSR); diff --git a/src/lib/types.h b/src/lib/types.h index b0e9e51ac..adb2268e3 100644 --- a/src/lib/types.h +++ b/src/lib/types.h @@ -148,4 +148,9 @@ enum Resolution { std::string resolution_to_string (Resolution); Resolution string_to_resolution (std::string); +enum Protocol { + PROTOCOL_SCP, + PROTOCOL_FTP +}; + #endif diff --git a/src/lib/upload_job.cc b/src/lib/upload_job.cc index 89ce5252d..44114b8a8 100644 --- a/src/lib/upload_job.cc +++ b/src/lib/upload_job.cc @@ -21,20 +21,14 @@ * @brief A job to copy DCPs to a server using libcurl. */ -#include -#include -#include -#include -#include -#include #include "compose.hpp" #include "upload_job.h" -#include "exceptions.h" #include "config.h" #include "log.h" #include "film.h" -#include "cross.h" #include "scp_uploader.h" +#include "curl_uploader.h" +#include #include "i18n.h" @@ -43,6 +37,7 @@ using std::string; using std::min; using boost::shared_ptr; +using boost::scoped_ptr; UploadJob::UploadJob (shared_ptr film) : Job (film) @@ -68,8 +63,17 @@ UploadJob::run () { LOG_GENERAL_NC (N_("Upload job starting")); - SCPUploader uploader (bind (&UploadJob::set_status, this, _1), bind (&UploadJob::set_progress, this, _1, false)); - uploader.upload (_film->dir (_film->dcp_name ())); + scoped_ptr uploader; + switch (Config::instance()->tms_protocol ()) { + case PROTOCOL_SCP: + uploader.reset (new SCPUploader (bind (&UploadJob::set_status, this, _1), bind (&UploadJob::set_progress, this, _1, false))); + break; + case PROTOCOL_FTP: + uploader.reset (new CurlUploader (bind (&UploadJob::set_status, this, _1), bind (&UploadJob::set_progress, this, _1, false))); + break; + } + + uploader->upload (_film->dir (_film->dcp_name ())); set_progress (1); set_status (N_("")); diff --git a/src/lib/uploader.cc b/src/lib/uploader.cc index b6b23ed8e..fe15c5d83 100644 --- a/src/lib/uploader.cc +++ b/src/lib/uploader.cc @@ -19,16 +19,19 @@ #include "uploader.h" #include "dcpomatic_assert.h" +#include "compose.hpp" + +#include "i18n.h" using std::string; using boost::shared_ptr; using boost::function; Uploader::Uploader (function set_status, function set_progress) - : _set_status (set_status) - , _set_progress (set_progress) + : _set_progress (set_progress) + , _set_status (set_status) { - + _set_status (_("connecting")); } boost::uintmax_t @@ -66,6 +69,7 @@ Uploader::upload_directory (boost::filesystem::path base, boost::filesystem::pat if (is_directory (i->path ())) { upload_directory (base, i->path (), transferred, total_size); } else { + _set_status (String::compose (_("copying %1"), i->path().leaf ())); upload_file (i->path (), remove_prefix (base, i->path ()), transferred, total_size); } } diff --git a/src/lib/uploader.h b/src/lib/uploader.h index fcb9c5045..d504ed0f8 100644 --- a/src/lib/uploader.h +++ b/src/lib/uploader.h @@ -17,6 +17,9 @@ */ +#ifndef DCPOMATIC_UPLOADER_H +#define DCPOMATIC_UPLOADER_H + #include #include #include @@ -34,11 +37,14 @@ protected: virtual void create_directory (boost::filesystem::path directory) = 0; virtual void upload_file (boost::filesystem::path from, boost::filesystem::path to, boost::uintmax_t& transferred, boost::uintmax_t total_size) = 0; - boost::function _set_status; boost::function _set_progress; private: void upload_directory (boost::filesystem::path base, boost::filesystem::path directory, boost::uintmax_t& transferred, boost::uintmax_t total_size); boost::uintmax_t count_file_sizes (boost::filesystem::path) const; boost::filesystem::path remove_prefix (boost::filesystem::path prefix, boost::filesystem::path target) const; + + boost::function _set_status; }; + +#endif diff --git a/src/lib/wscript b/src/lib/wscript index eb2238960..3457329e6 100644 --- a/src/lib/wscript +++ b/src/lib/wscript @@ -37,6 +37,7 @@ sources = """ content.cc content_factory.cc cross.cc + curl_uploader.cc data.cc dcp_content.cc dcp_content_type.cc diff --git a/src/wx/config_dialog.cc b/src/wx/config_dialog.cc index 86ad2dbe1..92a700d4d 100644 --- a/src/wx/config_dialog.cc +++ b/src/wx/config_dialog.cc @@ -910,6 +910,10 @@ private: table->AddGrowableCol (1, 1); _panel->GetSizer()->Add (table, 1, wxALL | wxEXPAND, _border); + add_label_to_sizer (table, _panel, _("Protocol"), true); + _tms_protocol = new wxChoice (_panel, wxID_ANY); + table->Add (_tms_protocol, 1, wxEXPAND); + add_label_to_sizer (table, _panel, _("IP address"), true); _tms_ip = new wxTextCtrl (_panel, wxID_ANY); table->Add (_tms_ip, 1, wxEXPAND); @@ -926,6 +930,10 @@ private: _tms_password = new wxTextCtrl (_panel, wxID_ANY); table->Add (_tms_password, 1, wxEXPAND); + _tms_protocol->Append (_("SCP (for AAM)")); + _tms_protocol->Append (_("FTP (for Dolby)")); + + _tms_protocol->Bind (wxEVT_COMMAND_CHOICE_SELECTED, boost::bind (&TMSPage::tms_protocol_changed, this)); _tms_ip->Bind (wxEVT_COMMAND_TEXT_UPDATED, boost::bind (&TMSPage::tms_ip_changed, this)); _tms_path->Bind (wxEVT_COMMAND_TEXT_UPDATED, boost::bind (&TMSPage::tms_path_changed, this)); _tms_user->Bind (wxEVT_COMMAND_TEXT_UPDATED, boost::bind (&TMSPage::tms_user_changed, this)); @@ -936,12 +944,18 @@ private: { Config* config = Config::instance (); + checked_set (_tms_protocol, config->tms_protocol ()); checked_set (_tms_ip, config->tms_ip ()); checked_set (_tms_path, config->tms_path ()); checked_set (_tms_user, config->tms_user ()); checked_set (_tms_password, config->tms_password ()); } + void tms_protocol_changed () + { + Config::instance()->set_tms_protocol (static_cast (_tms_protocol->GetSelection ())); + } + void tms_ip_changed () { Config::instance()->set_tms_ip (wx_to_std (_tms_ip->GetValue ())); @@ -962,6 +976,7 @@ private: Config::instance()->set_tms_password (wx_to_std (_tms_password->GetValue ())); } + wxChoice* _tms_protocol; wxTextCtrl* _tms_ip; wxTextCtrl* _tms_path; wxTextCtrl* _tms_user; -- 2.30.2