Fix audio analysis; make sure we don't decode video and let it pile up unwanted.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include "player.h"
21 #include "film.h"
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
33 #include "playlist.h"
34 #include "job.h"
35 #include "image.h"
36 #include "raw_image_proxy.h"
37 #include "ratio.h"
38 #include "log.h"
39 #include "scaler.h"
40 #include "render_subtitles.h"
41 #include "config.h"
42 #include "content_video.h"
43 #include "player_video.h"
44 #include "frame_rate_change.h"
45 #include "dcp_content.h"
46 #include "dcp_decoder.h"
47 #include "dcp_subtitle_content.h"
48 #include "dcp_subtitle_decoder.h"
49 #include <boost/foreach.hpp>
50 #include <stdint.h>
51 #include <algorithm>
52
53 #include "i18n.h"
54
55 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
56
57 using std::list;
58 using std::cout;
59 using std::min;
60 using std::max;
61 using std::min;
62 using std::vector;
63 using std::pair;
64 using std::map;
65 using std::make_pair;
66 using boost::shared_ptr;
67 using boost::weak_ptr;
68 using boost::dynamic_pointer_cast;
69 using boost::optional;
70
71 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
72         : _film (f)
73         , _playlist (p)
74         , _have_valid_pieces (false)
75         , _approximate_size (false)
76         , _ignore_video (false)
77 {
78         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
79         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
80         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
81         set_video_container_size (_film->frame_size ());
82 }
83
84 void
85 Player::setup_pieces ()
86 {
87         list<shared_ptr<Piece> > old_pieces = _pieces;
88         _pieces.clear ();
89
90         ContentList content = _playlist->content ();
91
92         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
93
94                 if (!(*i)->paths_valid ()) {
95                         continue;
96                 }
97                 
98                 shared_ptr<Decoder> decoder;
99                 optional<FrameRateChange> frc;
100
101                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
102                 DCPTime best_overlap_t;
103                 shared_ptr<VideoContent> best_overlap;
104                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
105                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
106                         if (!vc) {
107                                 continue;
108                         }
109                         
110                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
111                         if (overlap > best_overlap_t) {
112                                 best_overlap = vc;
113                                 best_overlap_t = overlap;
114                         }
115                 }
116
117                 optional<FrameRateChange> best_overlap_frc;
118                 if (best_overlap) {
119                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
120                 } else {
121                         /* No video overlap; e.g. if the DCP is just audio */
122                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
123                 }
124
125                 /* FFmpeg */
126                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
127                 if (fc) {
128                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
129                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
130                 }
131
132                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
133                 if (dc) {
134                         decoder.reset (new DCPDecoder (dc));
135                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
136                 }
137
138                 /* ImageContent */
139                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
140                 if (ic) {
141                         /* See if we can re-use an old ImageDecoder */
142                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
143                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
144                                 if (imd && imd->content() == ic) {
145                                         decoder = imd;
146                                 }
147                         }
148
149                         if (!decoder) {
150                                 decoder.reset (new ImageDecoder (ic));
151                         }
152
153                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
154                 }
155
156                 /* SndfileContent */
157                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
158                 if (sc) {
159                         decoder.reset (new SndfileDecoder (sc));
160                         frc = best_overlap_frc;
161                 }
162
163                 /* SubRipContent */
164                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
165                 if (rc) {
166                         decoder.reset (new SubRipDecoder (rc));
167                         frc = best_overlap_frc;
168                 }
169
170                 /* DCPSubtitleContent */
171                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
172                 if (dsc) {
173                         decoder.reset (new DCPSubtitleDecoder (dsc));
174                         frc = best_overlap_frc;
175                 }
176
177                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
178                 if (vd && _ignore_video) {
179                         vd->set_ignore_video ();
180                 }
181
182                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
183         }
184
185         _have_valid_pieces = true;
186 }
187
188 void
189 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
190 {
191         shared_ptr<Content> c = w.lock ();
192         if (!c) {
193                 return;
194         }
195
196         if (
197                 property == ContentProperty::POSITION ||
198                 property == ContentProperty::LENGTH ||
199                 property == ContentProperty::TRIM_START ||
200                 property == ContentProperty::TRIM_END ||
201                 property == ContentProperty::PATH ||
202                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
203                 property == DCPContentProperty::CAN_BE_PLAYED
204                 ) {
205                 
206                 _have_valid_pieces = false;
207                 Changed (frequent);
208
209         } else if (
210                 property == SubtitleContentProperty::USE_SUBTITLES ||
211                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
212                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
213                 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
214                 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
215                 property == VideoContentProperty::VIDEO_CROP ||
216                 property == VideoContentProperty::VIDEO_SCALE ||
217                 property == VideoContentProperty::VIDEO_FRAME_RATE
218                 ) {
219                 
220                 Changed (frequent);
221         }
222 }
223
224 /** @param already_resampled true if this data has already been through the chain up to the resampler */
225 void
226 Player::playlist_changed ()
227 {
228         _have_valid_pieces = false;
229         Changed (false);
230 }
231
232 void
233 Player::set_video_container_size (dcp::Size s)
234 {
235         _video_container_size = s;
236
237         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
238         _black_image->make_black ();
239 }
240
241 void
242 Player::film_changed (Film::Property p)
243 {
244         /* Here we should notice Film properties that affect our output, and
245            alert listeners that our output now would be different to how it was
246            last time we were run.
247         */
248
249         if (p == Film::SCALER || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
250                 Changed (false);
251         }
252 }
253
254 list<PositionImage>
255 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
256 {
257         list<PositionImage> all;
258         
259         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
260                 if (!i->image) {
261                         continue;
262                 }
263
264                 /* We will scale the subtitle up to fit _video_container_size */
265                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
266                 
267                 /* Then we need a corrective translation, consisting of two parts:
268                  *
269                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
270                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
271                  *
272                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
273                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
274                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
275                  *
276                  * Combining these two translations gives these expressions.
277                  */
278
279                 all.push_back (
280                         PositionImage (
281                                 i->image->scale (
282                                         scaled_size,
283                                         Scaler::from_id ("bicubic"),
284                                         i->image->pixel_format (),
285                                         true
286                                         ),
287                                 Position<int> (
288                                         rint (_video_container_size.width * i->rectangle.x),
289                                         rint (_video_container_size.height * i->rectangle.y)
290                                         )
291                                 )
292                         );
293         }
294
295         return all;
296 }
297
298 void
299 Player::set_approximate_size ()
300 {
301         _approximate_size = true;
302 }
303
304 shared_ptr<PlayerVideo>
305 Player::black_player_video_frame (DCPTime time) const
306 {
307         return shared_ptr<PlayerVideo> (
308                 new PlayerVideo (
309                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
310                         time,
311                         Crop (),
312                         optional<float> (),
313                         _video_container_size,
314                         _video_container_size,
315                         Scaler::from_id ("bicubic"),
316                         EYES_BOTH,
317                         PART_WHOLE,
318                         Config::instance()->colour_conversions().front().conversion
319                 )
320         );
321 }
322
323 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
324 list<shared_ptr<PlayerVideo> >
325 Player::get_video (DCPTime time, bool accurate)
326 {
327         if (!_have_valid_pieces) {
328                 setup_pieces ();
329         }
330
331         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
332                 time,
333                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
334                 );
335
336         list<shared_ptr<PlayerVideo> > pvf;
337
338         if (ov.empty ()) {
339                 /* No video content at this time */
340                 pvf.push_back (black_player_video_frame (time));
341         } else {
342                 /* Create a PlayerVideo from the content's video at this time */
343
344                 shared_ptr<Piece> piece = ov.back ();
345                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
346                 DCPOMATIC_ASSERT (decoder);
347                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
348                 DCPOMATIC_ASSERT (content);
349
350                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
351                 if (content_video.empty ()) {
352                         pvf.push_back (black_player_video_frame (time));
353                         return pvf;
354                 }
355                 
356                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size (), _approximate_size ? 4 : 1);
357                 if (_approximate_size) {
358                         image_size.width &= ~3;
359                         image_size.height &= ~3;
360                 }
361                 
362                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
363                         pvf.push_back (
364                                 shared_ptr<PlayerVideo> (
365                                         new PlayerVideo (
366                                                 i->image,
367                                                 content_video_to_dcp (piece, i->frame),
368                                                 content->crop (),
369                                                 content->fade (i->frame),
370                                                 image_size,
371                                                 _video_container_size,
372                                                 _film->scaler(),
373                                                 i->eyes,
374                                                 i->part,
375                                                 content->colour_conversion ()
376                                                 )
377                                         )
378                                 );
379                 }
380         }
381
382         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
383
384         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
385
386         list<PositionImage> sub_images;
387
388         /* Image subtitles */
389         list<PositionImage> c = transform_image_subtitles (ps.image);
390         copy (c.begin(), c.end(), back_inserter (sub_images));
391
392         /* Text subtitles (rendered to images) */
393         sub_images.push_back (render_subtitles (ps.text, _video_container_size));
394         
395         if (!sub_images.empty ()) {
396                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
397                         (*i)->set_subtitle (merge (sub_images));
398                 }
399         }       
400                 
401         return pvf;
402 }
403
404 shared_ptr<AudioBuffers>
405 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
406 {
407         if (!_have_valid_pieces) {
408                 setup_pieces ();
409         }
410
411         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
412
413         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
414         audio->make_silent ();
415         
416         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
417         if (ov.empty ()) {
418                 return audio;
419         }
420
421         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
422
423                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
424                 DCPOMATIC_ASSERT (content);
425                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
426                 DCPOMATIC_ASSERT (decoder);
427
428                 if (content->audio_frame_rate() == 0) {
429                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
430                          * audio stream).
431                          */
432                         continue;
433                 }
434
435                 /* The time that we should request from the content */
436                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
437                 DCPTime offset;
438                 if (request < DCPTime ()) {
439                         /* We went off the start of the content, so we will need to offset
440                            the stuff we get back.
441                         */
442                         offset = -request;
443                         request = DCPTime ();
444                 }
445
446                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
447
448                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
449                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, length_frames, accurate);
450
451                 /* Gain */
452                 if (content->audio_gain() != 0) {
453                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
454                         gain->apply_gain (content->audio_gain ());
455                         all->audio = gain;
456                 }
457
458                 /* Remap channels */
459                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
460                 dcp_mapped->make_silent ();
461                 AudioMapping map = content->audio_mapping ();
462                 for (int i = 0; i < map.content_channels(); ++i) {
463                         for (int j = 0; j < _film->audio_channels(); ++j) {
464                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
465                                         dcp_mapped->accumulate_channel (
466                                                 all->audio.get(),
467                                                 i,
468                                                 j,
469                                                 map.get (i, static_cast<dcp::Channel> (j))
470                                                 );
471                                 }
472                         }
473                 }
474                 
475                 all->audio = dcp_mapped;
476
477                 audio->accumulate_frames (
478                         all->audio.get(),
479                         content_frame - all->frame,
480                         offset.frames (_film->audio_frame_rate()),
481                         min (AudioFrame (all->audio->frames()), length_frames) - offset.frames (_film->audio_frame_rate ())
482                         );
483         }
484
485         return audio;
486 }
487
488 VideoFrame
489 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
490 {
491         /* s is the offset of t from the start position of this content */
492         DCPTime s = t - piece->content->position ();
493         s = DCPTime (max (DCPTime::Type (0), s.get ()));
494         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
495
496         /* Convert this to the content frame */
497         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) / piece->frc.factor ();
498 }
499
500 DCPTime
501 Player::content_video_to_dcp (shared_ptr<const Piece> piece, VideoFrame f) const
502 {
503         DCPTime t = DCPTime::from_frames (f * piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
504         if (t < DCPTime ()) {
505                 t = DCPTime ();
506         }
507
508         return t;
509 }
510
511 AudioFrame
512 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
513 {
514         /* s is the offset of t from the start position of this content */
515         DCPTime s = t - piece->content->position ();
516         s = DCPTime (max (DCPTime::Type (0), s.get ()));
517         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
518
519         /* Convert this to the content frame */
520         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
521 }
522
523 ContentTime
524 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
525 {
526         /* s is the offset of t from the start position of this content */
527         DCPTime s = t - piece->content->position ();
528         s = DCPTime (max (DCPTime::Type (0), s.get ()));
529         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
530
531         return ContentTime (s + piece->content->trim_start(), piece->frc);
532 }
533
534 void
535 PlayerStatistics::dump (shared_ptr<Log> log) const
536 {
537         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
538         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
539 }
540
541 PlayerStatistics const &
542 Player::statistics () const
543 {
544         return _statistics;
545 }
546
547 PlayerSubtitles
548 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
549 {
550         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
551
552         PlayerSubtitles ps (time, length);
553
554         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
555                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
556                 if (!subtitle_content->use_subtitles ()) {
557                         continue;
558                 }
559
560                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
561                 ContentTime const from = dcp_to_content_subtitle (*j, time);
562                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
563                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
564
565                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
566                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
567                         
568                         /* Apply content's subtitle offsets */
569                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
570                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
571
572                         /* Apply content's subtitle scale */
573                         i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
574                         i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
575
576                         /* Apply a corrective translation to keep the subtitle centred after that scale */
577                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
578                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
579                         
580                         ps.image.push_back (i->sub);
581                 }
582
583                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
584                 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
585                         BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
586                                 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
587                                 s.set_size (s.size() * max (subtitle_content->subtitle_x_scale(), subtitle_content->subtitle_y_scale()));
588                                 ps.text.push_back (s);
589                         }
590                 }
591         }
592
593         return ps;
594 }
595
596 list<shared_ptr<Font> >
597 Player::get_subtitle_fonts ()
598 {
599         if (!_have_valid_pieces) {
600                 setup_pieces ();
601         }
602
603         list<shared_ptr<Font> > fonts;
604         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
605                 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
606                 if (sc) {
607                         /* XXX: things may go wrong if there are duplicate font IDs
608                            with different font files.
609                         */
610                         list<shared_ptr<Font> > f = sc->fonts ();
611                         copy (f.begin(), f.end(), back_inserter (fonts));
612                 }
613         }
614
615         return fonts;
616 }
617
618 /** Set this player never to produce any video data */
619 void
620 Player::set_ignore_video ()
621 {
622         _ignore_video = true;
623 }