Post-export Analysis
[ardour.git] / libs / ardour / export_graph_builder.cc
1 /*
2   Copyright (C) 2008-2012 Paul Davis
3   Author: Sakari Bergen
4
5   This program is free software; you can redistribute it and/or modify
6   it under the terms of the GNU General Public License as published by
7   the Free Software Foundation; either version 2 of the License, or
8   (at your option) any later version.
9
10   This program is distributed in the hope that it will be useful,
11   but WITHOUT ANY WARRANTY; without even the implied warranty of
12   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13   GNU General Public License for more details.
14
15   You should have received a copy of the GNU General Public License
16   along with this program; if not, write to the Free Software
17   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18
19 */
20
21 #include "ardour/export_graph_builder.h"
22
23 #include <vector>
24
25 #include <glibmm/miscutils.h>
26
27 #include "audiographer/process_context.h"
28 #include "audiographer/general/chunker.h"
29 #include "audiographer/general/interleaver.h"
30 #include "audiographer/general/normalizer.h"
31 #include "audiographer/general/analyser.h"
32 #include "audiographer/general/peak_reader.h"
33 #include "audiographer/general/sample_format_converter.h"
34 #include "audiographer/general/sr_converter.h"
35 #include "audiographer/general/silence_trimmer.h"
36 #include "audiographer/general/threader.h"
37 #include "audiographer/sndfile/tmp_file.h"
38 #include "audiographer/sndfile/sndfile_writer.h"
39
40 #include "ardour/audioengine.h"
41 #include "ardour/export_channel_configuration.h"
42 #include "ardour/export_filename.h"
43 #include "ardour/export_format_specification.h"
44 #include "ardour/export_timespan.h"
45 #include "ardour/session_directory.h"
46 #include "ardour/sndfile_helpers.h"
47
48 #include "pbd/file_utils.h"
49 #include "pbd/cpus.h"
50
51 using namespace AudioGrapher;
52 using std::string;
53
54 namespace ARDOUR {
55
56 ExportGraphBuilder::ExportGraphBuilder (Session const & session)
57         : session (session)
58         , thread_pool (hardware_concurrency())
59 {
60         process_buffer_frames = session.engine().samples_per_cycle();
61 }
62
63 ExportGraphBuilder::~ExportGraphBuilder ()
64 {
65 }
66
67 int
68 ExportGraphBuilder::process (framecnt_t frames, bool last_cycle)
69 {
70         assert(frames <= process_buffer_frames);
71
72         for (ChannelMap::iterator it = channels.begin(); it != channels.end(); ++it) {
73                 Sample const * process_buffer = 0;
74                 it->first->read (process_buffer, frames);
75                 ConstProcessContext<Sample> context(process_buffer, frames, 1);
76                 if (last_cycle) { context().set_flag (ProcessContext<Sample>::EndOfInput); }
77                 it->second->process (context);
78         }
79
80         return 0;
81 }
82
83 bool
84 ExportGraphBuilder::process_normalize ()
85 {
86         for (std::list<Normalizer *>::iterator it = normalizers.begin(); it != normalizers.end(); /* ++ in loop */) {
87                 if ((*it)->process()) {
88                         it = normalizers.erase (it);
89                 } else {
90                         ++it;
91                 }
92         }
93
94         return normalizers.empty();
95 }
96
97 unsigned
98 ExportGraphBuilder::get_normalize_cycle_count() const
99 {
100         unsigned max = 0;
101         for (std::list<Normalizer *>::const_iterator it = normalizers.begin(); it != normalizers.end(); ++it) {
102                 max = std::max(max, (*it)->get_normalize_cycle_count());
103         }
104         return max;
105 }
106
107 void
108 ExportGraphBuilder::reset ()
109 {
110         timespan.reset();
111         channel_configs.clear ();
112         channels.clear ();
113         normalizers.clear ();
114         analysis_map.clear();
115 }
116
117 void
118 ExportGraphBuilder::cleanup (bool remove_out_files/*=false*/)
119 {
120         ChannelConfigList::iterator iter = channel_configs.begin();
121
122         while (iter != channel_configs.end() ) {
123                 iter->remove_children(remove_out_files);
124                 iter = channel_configs.erase(iter);
125         }
126 }
127
128 void
129 ExportGraphBuilder::set_current_timespan (boost::shared_ptr<ExportTimespan> span)
130 {
131         timespan = span;
132 }
133
134 void
135 ExportGraphBuilder::add_config (FileSpec const & config)
136 {
137         ExportChannelConfiguration::ChannelList const & channels =
138                 config.channel_config->get_channels();
139         for(ExportChannelConfiguration::ChannelList::const_iterator it = channels.begin();
140             it != channels.end(); ++it) {
141                 (*it)->set_max_buffer_size(process_buffer_frames);
142         }
143
144         // If the sample rate is "session rate", change it to the real value.
145         // However, we need to copy it to not change the config which is saved...
146         FileSpec new_config (config);
147         new_config.format.reset(new ExportFormatSpecification(*new_config.format, false));
148         if(new_config.format->sample_rate() == ExportFormatBase::SR_Session) {
149                 framecnt_t session_rate = session.nominal_frame_rate();
150                 new_config.format->set_sample_rate(ExportFormatBase::nearest_sample_rate(session_rate));
151         }
152
153
154         if (!new_config.channel_config->get_split ()) {
155                 add_split_config (new_config);
156                 return;
157         }
158
159         // Split channel configurations are split into several channel configurations,
160         // each corresponding to a file, at this stage
161         typedef std::list<boost::shared_ptr<ExportChannelConfiguration> > ConfigList;
162         ConfigList file_configs;
163         new_config.channel_config->configurations_for_files (file_configs);
164
165         unsigned chan = 1;
166         for (ConfigList::iterator it = file_configs.begin(); it != file_configs.end(); ++it, ++chan) {
167                 FileSpec copy = new_config;
168                 copy.channel_config = *it;
169
170                 copy.filename.reset (new ExportFilename (*copy.filename));
171                 copy.filename->include_channel = true;
172                 copy.filename->set_channel (chan);
173
174                 add_split_config (copy);
175         }
176 }
177
178 void
179 ExportGraphBuilder::get_analysis_results (AnalysisResults& results) {
180         for (AnalysisMap::iterator i = analysis_map.begin(); i != analysis_map.end(); ++i) {
181                 ExportAnalysisPtr p = i->second->result ();
182                 if (p) {
183                         results.insert (std::make_pair (i->first, p));
184                 }
185         }
186 }
187
188 void
189 ExportGraphBuilder::add_split_config (FileSpec const & config)
190 {
191         for (ChannelConfigList::iterator it = channel_configs.begin(); it != channel_configs.end(); ++it) {
192                 if (*it == config) {
193                         it->add_child (config);
194                         return;
195                 }
196         }
197
198         // No duplicate channel config found, create new one
199         channel_configs.push_back (new ChannelConfig (*this, config, channels));
200 }
201
202 /* Encoder */
203
204 template <>
205 boost::shared_ptr<AudioGrapher::Sink<Sample> >
206 ExportGraphBuilder::Encoder::init (FileSpec const & new_config)
207 {
208         config = new_config;
209         init_writer (float_writer);
210         return float_writer;
211 }
212
213 template <>
214 boost::shared_ptr<AudioGrapher::Sink<int> >
215 ExportGraphBuilder::Encoder::init (FileSpec const & new_config)
216 {
217         config = new_config;
218         init_writer (int_writer);
219         return int_writer;
220 }
221
222 template <>
223 boost::shared_ptr<AudioGrapher::Sink<short> >
224 ExportGraphBuilder::Encoder::init (FileSpec const & new_config)
225 {
226         config = new_config;
227         init_writer (short_writer);
228         return short_writer;
229 }
230
231 void
232 ExportGraphBuilder::Encoder::add_child (FileSpec const & new_config)
233 {
234         filenames.push_back (new_config.filename);
235 }
236
237 void
238 ExportGraphBuilder::Encoder::destroy_writer (bool delete_out_file)
239 {
240         if (delete_out_file ) {
241
242                 if (float_writer) {
243                         float_writer->close ();
244                 }
245
246                 if (int_writer) {
247                         int_writer->close ();
248                 }
249
250                 if (short_writer) {
251                         short_writer->close ();
252                 }
253
254                 if (std::remove(writer_filename.c_str() ) != 0) {
255                         std::cout << "Encoder::destroy_writer () : Error removing file: " << strerror(errno) << std::endl;
256                 }
257         }
258
259         float_writer.reset ();
260         int_writer.reset ();
261         short_writer.reset ();
262 }
263
264 bool
265 ExportGraphBuilder::Encoder::operator== (FileSpec const & other_config) const
266 {
267         return get_real_format (config) == get_real_format (other_config);
268 }
269
270 int
271 ExportGraphBuilder::Encoder::get_real_format (FileSpec const & config)
272 {
273         ExportFormatSpecification & format = *config.format;
274         return format.format_id() | format.sample_format() | format.endianness();
275 }
276
277 template<typename T>
278 void
279 ExportGraphBuilder::Encoder::init_writer (boost::shared_ptr<AudioGrapher::SndfileWriter<T> > & writer)
280 {
281         unsigned channels = config.channel_config->get_n_chans();
282         int format = get_real_format (config);
283         config.filename->set_channel_config(config.channel_config);
284         writer_filename = config.filename->get_path (config.format);
285
286         writer.reset (new AudioGrapher::SndfileWriter<T> (writer_filename, format, channels, config.format->sample_rate(), config.broadcast_info));
287         writer->FileWritten.connect_same_thread (copy_files_connection, boost::bind (&ExportGraphBuilder::Encoder::copy_files, this, _1));
288 }
289
290 void
291 ExportGraphBuilder::Encoder::copy_files (std::string orig_path)
292 {
293         while (filenames.size()) {
294                 ExportFilenamePtr & filename = filenames.front();
295                 PBD::copy_file (orig_path, filename->get_path (config.format).c_str());
296                 filenames.pop_front();
297         }
298 }
299
300 /* SFC */
301
302 ExportGraphBuilder::SFC::SFC (ExportGraphBuilder &parent, FileSpec const & new_config, framecnt_t max_frames)
303         : data_width(0)
304 {
305         config = new_config;
306         data_width = sndfile_data_width (Encoder::get_real_format (config));
307         unsigned channels = new_config.channel_config->get_n_chans();
308         analyser.reset (new Analyser (config.format->sample_rate(), channels, max_frames,
309                                 (framecnt_t) ceil (parent.timespan->get_length () * config.format->sample_rate () / (double) parent.session.nominal_frame_rate ())));
310         parent.add_analyser (config.filename->get_path (config.format), analyser);
311
312         if (data_width == 8 || data_width == 16) {
313                 short_converter = ShortConverterPtr (new SampleFormatConverter<short> (channels));
314                 short_converter->init (max_frames, config.format->dither_type(), data_width);
315                 add_child (config);
316                 analyser->add_output (short_converter);
317         } else if (data_width == 24 || data_width == 32) {
318                 int_converter = IntConverterPtr (new SampleFormatConverter<int> (channels));
319                 int_converter->init (max_frames, config.format->dither_type(), data_width);
320                 add_child (config);
321                 analyser->add_output (int_converter);
322         } else {
323                 int actual_data_width = 8 * sizeof(Sample);
324                 float_converter = FloatConverterPtr (new SampleFormatConverter<Sample> (channels));
325                 float_converter->init (max_frames, config.format->dither_type(), actual_data_width);
326                 add_child (config);
327                 analyser->add_output (float_converter);
328         }
329 }
330
331 ExportGraphBuilder::FloatSinkPtr
332 ExportGraphBuilder::SFC::sink ()
333 {
334         return analyser;
335 }
336
337 void
338 ExportGraphBuilder::SFC::add_child (FileSpec const & new_config)
339 {
340         for (boost::ptr_list<Encoder>::iterator it = children.begin(); it != children.end(); ++it) {
341                 if (*it == new_config) {
342                         it->add_child (new_config);
343                         return;
344                 }
345         }
346
347         children.push_back (new Encoder());
348         Encoder & encoder = children.back();
349
350         if (data_width == 8 || data_width == 16) {
351                 short_converter->add_output (encoder.init<short> (new_config));
352         } else if (data_width == 24 || data_width == 32) {
353                 int_converter->add_output (encoder.init<int> (new_config));
354         } else {
355                 float_converter->add_output (encoder.init<Sample> (new_config));
356         }
357 }
358
359 void
360 ExportGraphBuilder::SFC::remove_children (bool remove_out_files)
361 {
362         boost::ptr_list<Encoder>::iterator iter = children.begin ();
363
364         while (iter != children.end() ) {
365
366                 if (remove_out_files) {
367                         iter->destroy_writer(remove_out_files);
368                 }
369                 iter = children.erase (iter);
370         }
371 }
372
373 bool
374 ExportGraphBuilder::SFC::operator== (FileSpec const & other_config) const
375 {
376         return config.format->sample_format() == other_config.format->sample_format();
377 }
378
379 /* Normalizer */
380
381 ExportGraphBuilder::Normalizer::Normalizer (ExportGraphBuilder & parent, FileSpec const & new_config, framecnt_t /*max_frames*/)
382         : parent (parent)
383 {
384         std::string tmpfile_path = parent.session.session_directory().export_path();
385         tmpfile_path = Glib::build_filename(tmpfile_path, "XXXXXX");
386         std::vector<char> tmpfile_path_buf(tmpfile_path.size() + 1);
387         std::copy(tmpfile_path.begin(), tmpfile_path.end(), tmpfile_path_buf.begin());
388         tmpfile_path_buf[tmpfile_path.size()] = '\0';
389
390         config = new_config;
391         uint32_t const channels = config.channel_config->get_n_chans();
392         max_frames_out = 4086 - (4086 % channels); // TODO good chunk size
393
394         buffer.reset (new AllocatingProcessContext<Sample> (max_frames_out, channels));
395         peak_reader.reset (new PeakReader ());
396         normalizer.reset (new AudioGrapher::Normalizer (config.format->normalize_target()));
397         threader.reset (new Threader<Sample> (parent.thread_pool));
398
399         normalizer->alloc_buffer (max_frames_out);
400         normalizer->add_output (threader);
401
402         int format = ExportFormatBase::F_RAW | ExportFormatBase::SF_Float;
403         tmp_file.reset (new TmpFile<float> (&tmpfile_path_buf[0], format, channels, config.format->sample_rate()));
404         tmp_file->FileWritten.connect_same_thread (post_processing_connection,
405                                                    boost::bind (&Normalizer::start_post_processing, this));
406
407         add_child (new_config);
408
409         peak_reader->add_output (tmp_file);
410 }
411
412 ExportGraphBuilder::FloatSinkPtr
413 ExportGraphBuilder::Normalizer::sink ()
414 {
415         return peak_reader;
416 }
417
418 void
419 ExportGraphBuilder::Normalizer::add_child (FileSpec const & new_config)
420 {
421         for (boost::ptr_list<SFC>::iterator it = children.begin(); it != children.end(); ++it) {
422                 if (*it == new_config) {
423                         it->add_child (new_config);
424                         return;
425                 }
426         }
427
428         children.push_back (new SFC (parent, new_config, max_frames_out));
429         threader->add_output (children.back().sink());
430 }
431
432 void
433 ExportGraphBuilder::Normalizer::remove_children (bool remove_out_files)
434 {
435         boost::ptr_list<SFC>::iterator iter = children.begin ();
436
437         while (iter != children.end() ) {
438                 iter->remove_children (remove_out_files);
439                 iter = children.erase (iter);
440         }
441 }
442
443 bool
444 ExportGraphBuilder::Normalizer::operator== (FileSpec const & other_config) const
445 {
446         return config.format->normalize() == other_config.format->normalize() &&
447                 config.format->normalize_target() == other_config.format->normalize_target();
448 }
449
450 unsigned
451 ExportGraphBuilder::Normalizer::get_normalize_cycle_count() const
452 {
453         return static_cast<unsigned>(std::ceil(static_cast<float>(tmp_file->get_frames_written()) /
454                                                max_frames_out));
455 }
456
457 bool
458 ExportGraphBuilder::Normalizer::process()
459 {
460         framecnt_t frames_read = tmp_file->read (*buffer);
461         return frames_read != buffer->frames();
462 }
463
464 void
465 ExportGraphBuilder::Normalizer::start_post_processing()
466 {
467         normalizer->set_peak (peak_reader->get_peak());
468         tmp_file->seek (0, SEEK_SET);
469         tmp_file->add_output (normalizer);
470         parent.normalizers.push_back (this);
471 }
472
473 /* SRC */
474
475 ExportGraphBuilder::SRC::SRC (ExportGraphBuilder & parent, FileSpec const & new_config, framecnt_t max_frames)
476         : parent (parent)
477 {
478         config = new_config;
479         converter.reset (new SampleRateConverter (new_config.channel_config->get_n_chans()));
480         ExportFormatSpecification & format = *new_config.format;
481         converter->init (parent.session.nominal_frame_rate(), format.sample_rate(), format.src_quality());
482         max_frames_out = converter->allocate_buffers (max_frames);
483
484         add_child (new_config);
485 }
486
487 ExportGraphBuilder::FloatSinkPtr
488 ExportGraphBuilder::SRC::sink ()
489 {
490         return converter;
491 }
492
493 void
494 ExportGraphBuilder::SRC::add_child (FileSpec const & new_config)
495 {
496         if (new_config.format->normalize()) {
497                 add_child_to_list (new_config, normalized_children);
498         } else {
499                 add_child_to_list (new_config, children);
500         }
501 }
502
503 void
504 ExportGraphBuilder::SRC::remove_children (bool remove_out_files)
505 {
506         boost::ptr_list<SFC>::iterator sfc_iter = children.begin();
507
508         while (sfc_iter != children.end() ) {
509                 converter->remove_output (sfc_iter->sink() );
510                 sfc_iter->remove_children (remove_out_files);
511                 sfc_iter = children.erase (sfc_iter);
512         }
513
514         boost::ptr_list<Normalizer>::iterator norm_iter = normalized_children.begin();
515
516         while (norm_iter != normalized_children.end() ) {
517                 converter->remove_output (norm_iter->sink() );
518                 norm_iter->remove_children (remove_out_files);
519                 norm_iter = normalized_children.erase (norm_iter);
520         }
521
522 }
523
524 template<typename T>
525 void
526 ExportGraphBuilder::SRC::add_child_to_list (FileSpec const & new_config, boost::ptr_list<T> & list)
527 {
528         for (typename boost::ptr_list<T>::iterator it = list.begin(); it != list.end(); ++it) {
529                 if (*it == new_config) {
530                         it->add_child (new_config);
531                         return;
532                 }
533         }
534
535         list.push_back (new T (parent, new_config, max_frames_out));
536         converter->add_output (list.back().sink ());
537 }
538
539 bool
540 ExportGraphBuilder::SRC::operator== (FileSpec const & other_config) const
541 {
542         return config.format->sample_rate() == other_config.format->sample_rate();
543 }
544
545 /* SilenceHandler */
546 ExportGraphBuilder::SilenceHandler::SilenceHandler (ExportGraphBuilder & parent, FileSpec const & new_config, framecnt_t max_frames)
547         : parent (parent)
548 {
549         config = new_config;
550         max_frames_in = max_frames;
551         framecnt_t sample_rate = parent.session.nominal_frame_rate();
552
553         silence_trimmer.reset (new SilenceTrimmer<Sample>(max_frames_in));
554         silence_trimmer->set_trim_beginning (config.format->trim_beginning());
555         silence_trimmer->set_trim_end (config.format->trim_end());
556
557         framecnt_t sb = config.format->silence_beginning_at (parent.timespan->get_start(), sample_rate);
558         framecnt_t se = config.format->silence_end_at (parent.timespan->get_end(), sample_rate);
559
560         silence_trimmer->add_silence_to_beginning (sb);
561         silence_trimmer->add_silence_to_end (se);
562
563         add_child (new_config);
564 }
565
566 ExportGraphBuilder::FloatSinkPtr
567 ExportGraphBuilder::SilenceHandler::sink ()
568 {
569         return silence_trimmer;
570 }
571
572 void
573 ExportGraphBuilder::SilenceHandler::add_child (FileSpec const & new_config)
574 {
575         for (boost::ptr_list<SRC>::iterator it = children.begin(); it != children.end(); ++it) {
576                 if (*it == new_config) {
577                         it->add_child (new_config);
578                         return;
579                 }
580         }
581
582         children.push_back (new SRC (parent, new_config, max_frames_in));
583         silence_trimmer->add_output (children.back().sink());
584 }
585
586 void
587 ExportGraphBuilder::SilenceHandler::remove_children (bool remove_out_files)
588 {
589         boost::ptr_list<SRC>::iterator iter = children.begin();
590
591         while (iter != children.end() ) {
592                 silence_trimmer->remove_output (iter->sink() );
593                 iter->remove_children (remove_out_files);
594                 iter = children.erase (iter);
595         }
596 }
597
598 bool
599 ExportGraphBuilder::SilenceHandler::operator== (FileSpec const & other_config) const
600 {
601         ExportFormatSpecification & format = *config.format;
602         ExportFormatSpecification & other_format = *other_config.format;
603         return (format.trim_beginning() == other_format.trim_beginning()) &&
604                 (format.trim_end() == other_format.trim_end()) &&
605                 (format.silence_beginning_time() == other_format.silence_beginning_time()) &&
606                 (format.silence_end_time() == other_format.silence_end_time());
607 }
608
609 /* ChannelConfig */
610
611 ExportGraphBuilder::ChannelConfig::ChannelConfig (ExportGraphBuilder & parent, FileSpec const & new_config, ChannelMap & channel_map)
612         : parent (parent)
613 {
614         typedef ExportChannelConfiguration::ChannelList ChannelList;
615
616         config = new_config;
617
618         framecnt_t max_frames = parent.session.engine().samples_per_cycle();
619         interleaver.reset (new Interleaver<Sample> ());
620         interleaver->init (new_config.channel_config->get_n_chans(), max_frames);
621
622         // Make the chunk size divisible by the channel count
623         int chan_count = new_config.channel_config->get_n_chans();
624         max_frames_out = 8192;
625         if (chan_count > 0) {
626                 max_frames_out -= max_frames_out % chan_count;
627         }
628         chunker.reset (new Chunker<Sample> (max_frames_out));
629         interleaver->add_output(chunker);
630
631         ChannelList const & channel_list = config.channel_config->get_channels();
632         unsigned chan = 0;
633         for (ChannelList::const_iterator it = channel_list.begin(); it != channel_list.end(); ++it, ++chan) {
634                 ChannelMap::iterator map_it = channel_map.find (*it);
635                 if (map_it == channel_map.end()) {
636                         std::pair<ChannelMap::iterator, bool> result_pair =
637                                 channel_map.insert (std::make_pair (*it, IdentityVertexPtr (new IdentityVertex<Sample> ())));
638                         assert (result_pair.second);
639                         map_it = result_pair.first;
640                 }
641                 map_it->second->add_output (interleaver->input (chan));
642         }
643
644         add_child (new_config);
645 }
646
647 void
648 ExportGraphBuilder::ChannelConfig::add_child (FileSpec const & new_config)
649 {
650         assert (*this == new_config);
651
652         for (boost::ptr_list<SilenceHandler>::iterator it = children.begin(); it != children.end(); ++it) {
653                 if (*it == new_config) {
654                         it->add_child (new_config);
655                         return;
656                 }
657         }
658
659         children.push_back (new SilenceHandler (parent, new_config, max_frames_out));
660         chunker->add_output (children.back().sink ());
661 }
662
663 void
664 ExportGraphBuilder::ChannelConfig::remove_children (bool remove_out_files)
665 {
666         boost::ptr_list<SilenceHandler>::iterator iter = children.begin();
667
668         while(iter != children.end() ) {
669
670                 chunker->remove_output (iter->sink ());
671                 iter->remove_children (remove_out_files);
672                 iter = children.erase(iter);
673         }
674 }
675
676 bool
677 ExportGraphBuilder::ChannelConfig::operator== (FileSpec const & other_config) const
678 {
679         return config.channel_config == other_config.channel_config;
680 }
681
682 } // namespace ARDOUR