libs/ardour/import.cc

   1 /*
   2  * Copyright (C) 2000-2017 Paul Davis <paul@linuxaudiosystems.com>
   3  * Copyright (C) 2006-2016 David Robillard <d@drobilla.net>
   4  * Copyright (C) 2007-2012 Tim Mayberry <mojofunk@gmail.com>
   5  * Copyright (C) 2009-2012 Carl Hetherington <carl@carlh.net>
   6  * Copyright (C) 2014-2018 Robin Gareus <robin@gareus.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22
  23 #ifdef WAF_BUILD
  24 #include "libardour-config.h"
  25 #endif
  26
  27 #include <cstdio>
  28 #include <cstdlib>
  29 #include <string>
  30 #include <climits>
  31 #include <cerrno>
  32 #include <unistd.h>
  33 #include <sys/stat.h>
  34 #include <time.h>
  35 #include <stdint.h>
  36
  37 #include <sndfile.h>
  38 #include <samplerate.h>
  39
  40 #include "pbd/gstdio_compat.h"
  41 #include <glibmm.h>
  42
  43 #include <boost/scoped_array.hpp>
  44 #include <boost/scoped_ptr.hpp>
  45 #include <boost/shared_array.hpp>
  46
  47 #include "pbd/basename.h"
  48 #include "pbd/convert.h"
  49
  50 #include "evoral/SMF.hpp"
  51
  52 #include "ardour/analyser.h"
  53 #include "ardour/ardour.h"
  54 #include "ardour/audioengine.h"
  55 #include "ardour/audioregion.h"
  56 #include "ardour/import_status.h"
  57 #include "ardour/region_factory.h"
  58 #include "ardour/resampled_source.h"
  59 #include "ardour/runtime_functions.h"
  60 #include "ardour/session.h"
  61 #include "ardour/session_directory.h"
  62 #include "ardour/smf_source.h"
  63 #include "ardour/sndfile_helpers.h"
  64 #include "ardour/sndfileimportable.h"
  65 #include "ardour/sndfilesource.h"
  66 #include "ardour/source_factory.h"
  67 #include "ardour/tempo.h"
  68
  69 #ifdef HAVE_COREAUDIO
  70 #include "ardour/caimportable.h"
  71 #endif
  72
  73 #include "pbd/i18n.h"
  74
  75 using namespace std;
  76 using namespace ARDOUR;
  77 using namespace PBD;
  78
  79 static boost::shared_ptr<ImportableSource>
  80 open_importable_source (const string& path, samplecnt_t samplerate, ARDOUR::SrcQuality quality)
  81 {
  82         /* try libsndfile first, because it can get BWF info from .wav, which ExtAudioFile cannot.
  83            We don't necessarily need that information in an ImportableSource, but it keeps the
  84            logic the same as in SourceFactory::create()
  85         */
  86
  87         try {
  88                 boost::shared_ptr<SndFileImportableSource> source(new SndFileImportableSource(path));
  89
  90                 if (source->samplerate() == samplerate) {
  91                         return source;
  92                 }
  93
  94                 /* rewrap as a resampled source */
  95
  96                 return boost::shared_ptr<ImportableSource>(new ResampledImportableSource(source, samplerate, quality));
  97         }
  98
  99         catch (...) {
 100
 101 #ifdef HAVE_COREAUDIO
 102
 103                 /* libsndfile failed, see if we can use CoreAudio to handle the IO */
 104
 105                 CAImportableSource* src = new CAImportableSource(path);
 106                 boost::shared_ptr<CAImportableSource> source (src);
 107
 108                 if (source->samplerate() == samplerate) {
 109                         return source;
 110                 }
 111
 112                 /* rewrap as a resampled source */
 113
 114                 return boost::shared_ptr<ImportableSource>(new ResampledImportableSource(source, samplerate, quality));
 115
 116 #else
 117                 throw; // rethrow
 118 #endif
 119
 120         }
 121 }
 122
 123 vector<string>
 124 Session::get_paths_for_new_sources (bool /*allow_replacing*/, const string& import_file_path, uint32_t channels,
 125                                     vector<string> const & smf_track_names)
 126
 127 {
 128         vector<string> new_paths;
 129         const string basename = basename_nosuffix (import_file_path);
 130
 131         for (uint32_t n = 0; n < channels; ++n) {
 132
 133                 const DataType type = SMFSource::safe_midi_file_extension (import_file_path) ? DataType::MIDI : DataType::AUDIO;
 134                 string filepath;
 135
 136                 switch (type) {
 137                 case DataType::MIDI:
 138                         assert (smf_track_names.empty() || smf_track_names.size() == channels);
 139                         if (channels > 1) {
 140                                 string mchn_name;
 141                                 if (smf_track_names.empty() || smf_track_names[n].empty()) {
 142                                         mchn_name = string_compose ("%1-t%2", basename, n);
 143                                 } else {
 144                                         mchn_name = string_compose ("%1-%2", basename, smf_track_names[n]);
 145                                 }
 146                                 filepath = new_midi_source_path (mchn_name);
 147                         } else {
 148                                 filepath = new_midi_source_path (basename);
 149                         }
 150                         break;
 151                 case DataType::AUDIO:
 152                         filepath = new_audio_source_path (basename, channels, n, false, false);
 153                         break;
 154                 }
 155
 156                 if (filepath.empty()) {
 157                         error << string_compose (_("Cannot find new filename for imported file %1"), import_file_path) << endmsg;
 158                         return vector<string>();
 159                 }
 160
 161                 new_paths.push_back (filepath);
 162         }
 163
 164         return new_paths;
 165 }
 166
 167 static bool
 168 map_existing_mono_sources (const vector<string>& new_paths, Session& /*sess*/,
 169                            uint32_t /*samplerate*/, vector<boost::shared_ptr<Source> >& newfiles, Session *session)
 170 {
 171         for (vector<string>::const_iterator i = new_paths.begin();
 172              i != new_paths.end(); ++i)
 173         {
 174                 boost::shared_ptr<Source> source = session->audio_source_by_path_and_channel(*i, 0);
 175
 176                 if (source == 0) {
 177                         error << string_compose(_("Could not find a source for %1 even though we are updating this file!"), (*i)) << endl;
 178                         return false;
 179                 }
 180
 181                 newfiles.push_back(boost::dynamic_pointer_cast<Source>(source));
 182         }
 183         return true;
 184 }
 185
 186 static bool
 187 create_mono_sources_for_writing (const vector<string>& new_paths,
 188                                  Session& sess, uint32_t samplerate,
 189                                  vector<boost::shared_ptr<Source> >& newfiles,
 190                                  samplepos_t natural_position)
 191 {
 192         for (vector<string>::const_iterator i = new_paths.begin(); i != new_paths.end(); ++i) {
 193
 194                 boost::shared_ptr<Source> source;
 195
 196                 try {
 197                         const DataType type = SMFSource::safe_midi_file_extension (*i) ? DataType::MIDI : DataType::AUDIO;
 198
 199                         source = SourceFactory::createWritable (type, sess,
 200                                                                 i->c_str(),
 201                                                                 false, // destructive
 202                                                                 samplerate);
 203                 }
 204
 205                 catch (const failed_constructor& err) {
 206                         error << string_compose (_("Unable to create file %1 during import"), *i) << endmsg;
 207                         return false;
 208                 }
 209
 210                 newfiles.push_back(boost::dynamic_pointer_cast<Source>(source));
 211
 212                 /* for audio files, reset the timeline position so that any BWF-ish
 213                    information in the original files we are importing from is maintained.
 214                 */
 215
 216                 boost::shared_ptr<AudioFileSource> afs;
 217                 if ((afs = boost::dynamic_pointer_cast<AudioFileSource>(source)) != 0) {
 218                         afs->set_natural_position (natural_position);
 219                 }
 220         }
 221         return true;
 222 }
 223
 224 static string
 225 compose_status_message (const string& path,
 226                         uint32_t file_samplerate,
 227                         uint32_t session_samplerate,
 228                         uint32_t /* current_file */,
 229                         uint32_t /* total_files */)
 230 {
 231         if (file_samplerate != session_samplerate) {
 232                 return string_compose (_("Resampling %1 from %2kHz to %3kHz"),
 233                                        Glib::path_get_basename (path),
 234                                        file_samplerate/1000.0f,
 235                                        session_samplerate/1000.0f);
 236         }
 237
 238         return string_compose (_("Copying %1"), Glib::path_get_basename (path));
 239 }
 240
 241 static void
 242 write_audio_data_to_new_files (ImportableSource* source, ImportStatus& status,
 243                                vector<boost::shared_ptr<Source> >& newfiles)
 244 {
 245         const samplecnt_t nframes = ResampledImportableSource::blocksize;
 246         boost::shared_ptr<AudioFileSource> afs;
 247         uint32_t channels = source->channels();
 248         if (channels == 0) {
 249                 return;
 250         }
 251
 252         boost::scoped_array<float> data(new float[nframes * channels]);
 253         vector<boost::shared_array<Sample> > channel_data;
 254
 255         for (uint32_t n = 0; n < channels; ++n) {
 256                 channel_data.push_back(boost::shared_array<Sample>(new Sample[nframes]));
 257         }
 258
 259         float gain = 1;
 260
 261         boost::shared_ptr<AudioSource> s = boost::dynamic_pointer_cast<AudioSource> (newfiles[0]);
 262         assert (s);
 263
 264         status.progress = 0.0f;
 265         float progress_multiplier = 1;
 266         float progress_base = 0;
 267         const float progress_length = source->ratio() * source->length();
 268
 269         if (!source->clamped_at_unity() && s->clamped_at_unity()) {
 270
 271                 /* The source we are importing from can return sample values with a magnitude greater than 1,
 272                    and the file we are writing the imported data to cannot handle such values.  Compute the gain
 273                    factor required to normalize the input sources to have a magnitude of less than 1.
 274                 */
 275
 276                 float peak = 0;
 277                 uint32_t read_count = 0;
 278
 279                 while (!status.cancel) {
 280                         samplecnt_t const nread = source->read (data.get(), nframes * channels);
 281                         if (nread == 0) {
 282                                 break;
 283                         }
 284
 285                         peak = compute_peak (data.get(), nread, peak);
 286
 287                         read_count += nread / channels;
 288                         status.progress = 0.5 * read_count / progress_length;
 289                 }
 290
 291                 if (peak >= 1) {
 292                         /* we are out of range: compute a gain to fix it */
 293                         gain = (1 - FLT_EPSILON) / peak;
 294                 }
 295
 296                 source->seek (0);
 297                 progress_multiplier = 0.5;
 298                 progress_base = 0.5;
 299         }
 300
 301         samplecnt_t read_count = 0;
 302
 303         while (!status.cancel) {
 304
 305                 samplecnt_t nread, nfread;
 306                 uint32_t x;
 307                 uint32_t chn;
 308
 309                 if ((nread = source->read (data.get(), nframes * channels)) == 0) {
 310 #ifdef PLATFORM_WINDOWS
 311                         /* Flush the data once we've finished importing the file. Windows can  */
 312                         /* cache the data for very long periods of time (perhaps not writing   */
 313                         /* it to disk until Ardour closes). So let's force it to flush now.    */
 314                         for (chn = 0; chn < channels; ++chn)
 315                                 if ((afs = boost::dynamic_pointer_cast<AudioFileSource>(newfiles[chn])) != 0)
 316                                         afs->flush ();
 317 #endif
 318                         break;
 319                 }
 320
 321                 if (gain != 1) {
 322                         /* here is the gain fix for out-of-range sample values that we computed earlier */
 323                         apply_gain_to_buffer (data.get(), nread, gain);
 324                 }
 325
 326                 nfread = nread / channels;
 327
 328                 /* de-interleave */
 329
 330                 for (chn = 0; chn < channels; ++chn) {
 331
 332                         samplecnt_t n;
 333                         for (x = chn, n = 0; n < nfread; x += channels, ++n) {
 334                                 channel_data[chn][n] = (Sample) data[x];
 335                         }
 336                 }
 337
 338                 /* flush to disk */
 339
 340                 for (chn = 0; chn < channels; ++chn) {
 341                         if ((afs = boost::dynamic_pointer_cast<AudioFileSource>(newfiles[chn])) != 0) {
 342                                 afs->write (channel_data[chn].get(), nfread);
 343                         }
 344                 }
 345
 346                 read_count += nfread;
 347                 status.progress = progress_base + progress_multiplier * read_count / progress_length;
 348         }
 349 }
 350
 351 static void
 352 write_midi_data_to_new_files (Evoral::SMF* source, ImportStatus& status,
 353                               vector<boost::shared_ptr<Source> >& newfiles,
 354                               bool split_type0)
 355 {
 356         uint32_t buf_size = 4;
 357         uint8_t* buf      = (uint8_t*) malloc (buf_size);
 358
 359         status.progress = 0.0f;
 360         uint16_t num_tracks;
 361         bool type0 = source->is_type0 () && split_type0;
 362         const std::set<uint8_t>& chn = source->channels ();
 363
 364         if (type0) {
 365                 num_tracks = source->channels().size();
 366         } else {
 367                 num_tracks = source->num_tracks();
 368         }
 369         assert (newfiles.size() == num_tracks);
 370
 371         try {
 372                 vector<boost::shared_ptr<Source> >::iterator s = newfiles.begin();
 373                 std::set<uint8_t>::const_iterator cur_chan = chn.begin();
 374
 375                 for (unsigned i = 1; i <= num_tracks; ++i) {
 376
 377                         boost::shared_ptr<SMFSource> smfs = boost::dynamic_pointer_cast<SMFSource> (*s);
 378
 379                         Glib::Threads::Mutex::Lock source_lock(smfs->mutex());
 380
 381                         smfs->drop_model (source_lock);
 382                         if (type0) {
 383                                 source->seek_to_start ();
 384                         } else {
 385                                 source->seek_to_track (i);
 386                         }
 387
 388                         uint64_t t       = 0;
 389                         uint32_t delta_t = 0;
 390                         uint32_t size    = 0;
 391                         bool first = true;
 392
 393                         while (!status.cancel) {
 394                                 gint note_id_ignored; // imported files either don't have NoteID's or we ignore them.
 395
 396                                 size = buf_size;
 397
 398                                 int ret = source->read_event (&delta_t, &size, &buf, &note_id_ignored);
 399
 400                                 if (size > buf_size) {
 401                                         buf_size = size;
 402                                 }
 403
 404                                 if (ret < 0) { // EOT
 405                                         break;
 406                                 }
 407
 408                                 t += delta_t;
 409
 410                                 if (ret == 0) { // Meta
 411                                         continue;
 412                                 }
 413
 414                                 // type-0 files separate by channel
 415                                 if (type0) {
 416                                         uint8_t type = buf[0] & 0xf0;
 417                                         uint8_t chan = buf[0] & 0x0f;
 418                                         if (type >= 0x80 && type <= 0xE0) {
 419                                                 if (chan != *cur_chan) {
 420                                                         continue;
 421                                                 }
 422                                         }
 423                                 }
 424
 425                                 if (first) {
 426                                         smfs->mark_streaming_write_started (source_lock);
 427                                         first = false;
 428                                 }
 429
 430                                 smfs->append_event_beats(
 431                                         source_lock,
 432                                         Evoral::Event<Temporal::Beats>(
 433                                                 Evoral::MIDI_EVENT,
 434                                                 Temporal::Beats::ticks_at_rate(t, source->ppqn()),
 435                                                 size,
 436                                                 buf));
 437
 438                                 if (status.progress < 0.99) {
 439                                         status.progress += 0.01;
 440                                 }
 441                         }
 442
 443                         if (!first) {
 444
 445                                 /* we wrote something */
 446
 447                                 const samplepos_t     pos          = 0;
 448                                 const Temporal::Beats  length_beats = Temporal::Beats::ticks_at_rate(t, source->ppqn());
 449                                 BeatsSamplesConverter converter(smfs->session().tempo_map(), pos);
 450                                 smfs->update_length(pos + converter.to(length_beats.round_up_to_beat()));
 451                                 smfs->mark_streaming_write_completed (source_lock);
 452
 453                                 if (status.cancel) {
 454                                         break;
 455                                 }
 456                         } else {
 457                                 info << string_compose (_("Track %1 of %2 contained no usable MIDI data"), i, num_tracks) << endmsg;
 458                         }
 459
 460                         ++s; // next source
 461                         if (type0) {
 462                                 ++cur_chan;
 463                         }
 464                 }
 465
 466         } catch (exception& e) {
 467                 error << string_compose (_("MIDI file could not be written (best guess: %1)"), e.what()) << endmsg;
 468         }
 469
 470         if (buf) {
 471                 free (buf);
 472         }
 473 }
 474
 475 static void
 476 remove_file_source (boost::shared_ptr<Source> source)
 477 {
 478         boost::shared_ptr<FileSource> fs = boost::dynamic_pointer_cast<FileSource> (source);
 479
 480         fs->DropReferences ();
 481
 482         if (fs) {
 483                 ::g_unlink (fs->path().c_str());
 484         }
 485 }
 486
 487 // This function is still unable to cleanly update an existing source, even though
 488 // it is possible to set the ImportStatus flag accordingly. The functinality
 489 // is disabled at the GUI until the Source implementations are able to provide
 490 // the necessary API.
 491 void
 492 Session::import_files (ImportStatus& status)
 493 {
 494         typedef vector<boost::shared_ptr<Source> > Sources;
 495         Sources all_new_sources;
 496         boost::shared_ptr<AudioFileSource> afs;
 497         boost::shared_ptr<SMFSource> smfs;
 498         uint32_t channels = 0;
 499         vector<string> smf_names;
 500
 501         status.sources.clear ();
 502
 503         for (vector<string>::const_iterator p = status.paths.begin();
 504              p != status.paths.end() && !status.cancel;
 505              ++p)
 506         {
 507                 boost::shared_ptr<ImportableSource> source;
 508
 509                 const DataType type = SMFSource::safe_midi_file_extension (*p) ? DataType::MIDI : DataType::AUDIO;
 510                 boost::scoped_ptr<Evoral::SMF> smf_reader;
 511
 512                 if (type == DataType::AUDIO) {
 513                         try {
 514                                 source = open_importable_source (*p, sample_rate(), status.quality);
 515                                 channels = source->channels();
 516                         } catch (const failed_constructor& err) {
 517                                 error << string_compose(_("Import: cannot open input sound file \"%1\""), (*p)) << endmsg;
 518                                 status.done = status.cancel = true;
 519                                 return;
 520                         }
 521
 522                 } else {
 523                         try {
 524                                 smf_reader.reset (new Evoral::SMF());
 525
 526                                 if (smf_reader->open(*p)) {
 527                                         throw Evoral::SMF::FileError (*p);
 528                                 }
 529
 530                                 if (smf_reader->is_type0 () && status.split_midi_channels) {
 531                                         channels = smf_reader->channels().size();
 532                                 } else {
 533                                         channels = smf_reader->num_tracks();
 534                                         switch (status.midi_track_name_source) {
 535                                         case SMFTrackNumber:
 536                                                 break;
 537                                         case SMFTrackName:
 538                                                 smf_reader->track_names (smf_names);
 539                                                 break;
 540                                         case SMFInstrumentName:
 541                                                 smf_reader->instrument_names (smf_names);
 542                                                 break;
 543                                         }
 544                                 }
 545                         } catch (...) {
 546                                 error << _("Import: error opening MIDI file") << endmsg;
 547                                 status.done = status.cancel = true;
 548                                 return;
 549                         }
 550                 }
 551
 552                 if (channels == 0) {
 553                         error << _("Import: file contains no channels.") << endmsg;
 554                         continue;
 555                 }
 556
 557                 vector<string> new_paths = get_paths_for_new_sources (status.replace_existing_source, *p, channels, smf_names);
 558                 Sources newfiles;
 559                 samplepos_t natural_position = source ? source->natural_position() : 0;
 560
 561
 562                 if (status.replace_existing_source) {
 563                         fatal << "THIS IS NOT IMPLEMENTED YET, IT SHOULD NEVER GET CALLED!!! DYING!" << endmsg;
 564                         status.cancel = !map_existing_mono_sources (new_paths, *this, sample_rate(), newfiles, this);
 565                 } else {
 566                         status.cancel = !create_mono_sources_for_writing (new_paths, *this, sample_rate(), newfiles, natural_position);
 567                 }
 568
 569                 // copy on cancel/failure so that any files that were created will be removed below
 570                 std::copy (newfiles.begin(), newfiles.end(), std::back_inserter(all_new_sources));
 571
 572                 if (status.cancel) {
 573                         break;
 574                 }
 575
 576                 for (Sources::iterator i = newfiles.begin(); i != newfiles.end(); ++i) {
 577                         if ((afs = boost::dynamic_pointer_cast<AudioFileSource>(*i)) != 0) {
 578                                 afs->prepare_for_peakfile_writes ();
 579                         }
 580                 }
 581
 582                 if (source) { // audio
 583                         status.doing_what = compose_status_message (*p, source->samplerate(),
 584                                                                     sample_rate(), status.current, status.total);
 585                         write_audio_data_to_new_files (source.get(), status, newfiles);
 586                 } else if (smf_reader) { // midi
 587                         status.doing_what = string_compose(_("Loading MIDI file %1"), *p);
 588                         write_midi_data_to_new_files (smf_reader.get(), status, newfiles, status.split_midi_channels);
 589                 }
 590
 591                 ++status.current;
 592                 status.progress = 0;
 593         }
 594
 595         if (!status.cancel) {
 596                 struct tm* now;
 597                 time_t xnow;
 598                 time (&xnow);
 599                 now = localtime (&xnow);
 600                 status.freeze = true;
 601
 602                 /* flush the final length(s) to the header(s) */
 603
 604                 for (Sources::iterator x = all_new_sources.begin(); x != all_new_sources.end(); ) {
 605
 606                         if ((afs = boost::dynamic_pointer_cast<AudioFileSource>(*x)) != 0) {
 607                                 afs->update_header((*x)->natural_position(), *now, xnow);
 608                                 afs->done_with_peakfile_writes ();
 609
 610                                 /* now that there is data there, requeue the file for analysis */
 611
 612                                 if (Config->get_auto_analyse_audio()) {
 613                                         Analyser::queue_source_for_analysis (boost::static_pointer_cast<Source>(*x), false);
 614                                 }
 615                         }
 616
 617                         /* imported, copied files cannot be written or removed
 618                          */
 619
 620                         boost::shared_ptr<FileSource> fs = boost::dynamic_pointer_cast<FileSource>(*x);
 621                         if (fs) {
 622                                 /* Only audio files should be marked as
 623                                    immutable - we may need to rewrite MIDI
 624                                    files at any time.
 625                                 */
 626                                 if (boost::dynamic_pointer_cast<AudioFileSource> (fs)) {
 627                                         fs->mark_immutable ();
 628                                 } else {
 629                                         fs->mark_immutable_except_write ();
 630                                 }
 631                                 fs->mark_nonremovable ();
 632                         }
 633
 634                         /* don't create tracks for empty MIDI sources (channels) */
 635
 636                         if ((smfs = boost::dynamic_pointer_cast<SMFSource>(*x)) != 0 && smfs->is_empty()) {
 637                                 x = all_new_sources.erase(x);
 638                         } else {
 639                                 ++x;
 640                         }
 641                 }
 642
 643                 /* save state so that we don't lose these new Sources */
 644
 645                 save_state (_name);
 646
 647                 std::copy (all_new_sources.begin(), all_new_sources.end(), std::back_inserter(status.sources));
 648         } else {
 649                 try {
 650                         std::for_each (all_new_sources.begin(), all_new_sources.end(), remove_file_source);
 651                 } catch (...) {
 652                         error << _("Failed to remove some files after failed/cancelled import operation") << endmsg;
 653                 }
 654
 655         }
 656
 657         status.done = true;
 658 }
 659