Handle multiple audio streams in a single piece of content
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include "player.h"
21 #include "film.h"
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
33 #include "playlist.h"
34 #include "job.h"
35 #include "image.h"
36 #include "raw_image_proxy.h"
37 #include "ratio.h"
38 #include "log.h"
39 #include "render_subtitles.h"
40 #include "config.h"
41 #include "content_video.h"
42 #include "player_video.h"
43 #include "frame_rate_change.h"
44 #include "dcp_content.h"
45 #include "dcp_decoder.h"
46 #include "dcp_subtitle_content.h"
47 #include "dcp_subtitle_decoder.h"
48 #include <boost/foreach.hpp>
49 #include <stdint.h>
50 #include <algorithm>
51
52 #include "i18n.h"
53
54 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
55
56 using std::list;
57 using std::cout;
58 using std::min;
59 using std::max;
60 using std::min;
61 using std::vector;
62 using std::pair;
63 using std::map;
64 using std::make_pair;
65 using boost::shared_ptr;
66 using boost::weak_ptr;
67 using boost::dynamic_pointer_cast;
68 using boost::optional;
69
70 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
71         : _film (f)
72         , _playlist (p)
73         , _have_valid_pieces (false)
74         , _ignore_video (false)
75 {
76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
79         set_video_container_size (_film->frame_size ());
80 }
81
82 void
83 Player::setup_pieces ()
84 {
85         list<shared_ptr<Piece> > old_pieces = _pieces;
86         _pieces.clear ();
87
88         ContentList content = _playlist->content ();
89
90         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
91
92                 if (!(*i)->paths_valid ()) {
93                         continue;
94                 }
95                 
96                 shared_ptr<Decoder> decoder;
97                 optional<FrameRateChange> frc;
98
99                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
100                 DCPTime best_overlap_t;
101                 shared_ptr<VideoContent> best_overlap;
102                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
103                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
104                         if (!vc) {
105                                 continue;
106                         }
107                         
108                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
109                         if (overlap > best_overlap_t) {
110                                 best_overlap = vc;
111                                 best_overlap_t = overlap;
112                         }
113                 }
114
115                 optional<FrameRateChange> best_overlap_frc;
116                 if (best_overlap) {
117                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
118                 } else {
119                         /* No video overlap; e.g. if the DCP is just audio */
120                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
121                 }
122
123                 /* FFmpeg */
124                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
125                 if (fc) {
126                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
127                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
128                 }
129
130                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
131                 if (dc) {
132                         decoder.reset (new DCPDecoder (dc));
133                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
134                 }
135
136                 /* ImageContent */
137                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
138                 if (ic) {
139                         /* See if we can re-use an old ImageDecoder */
140                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
141                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
142                                 if (imd && imd->content() == ic) {
143                                         decoder = imd;
144                                 }
145                         }
146
147                         if (!decoder) {
148                                 decoder.reset (new ImageDecoder (ic));
149                         }
150
151                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
152                 }
153
154                 /* SndfileContent */
155                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
156                 if (sc) {
157                         decoder.reset (new SndfileDecoder (sc));
158                         frc = best_overlap_frc;
159                 }
160
161                 /* SubRipContent */
162                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
163                 if (rc) {
164                         decoder.reset (new SubRipDecoder (rc));
165                         frc = best_overlap_frc;
166                 }
167
168                 /* DCPSubtitleContent */
169                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
170                 if (dsc) {
171                         decoder.reset (new DCPSubtitleDecoder (dsc));
172                         frc = best_overlap_frc;
173                 }
174
175                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
176                 if (vd && _ignore_video) {
177                         vd->set_ignore_video ();
178                 }
179
180                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
181         }
182
183         _have_valid_pieces = true;
184 }
185
186 void
187 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
188 {
189         shared_ptr<Content> c = w.lock ();
190         if (!c) {
191                 return;
192         }
193
194         if (
195                 property == ContentProperty::POSITION ||
196                 property == ContentProperty::LENGTH ||
197                 property == ContentProperty::TRIM_START ||
198                 property == ContentProperty::TRIM_END ||
199                 property == ContentProperty::PATH ||
200                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
201                 property == DCPContentProperty::CAN_BE_PLAYED
202                 ) {
203                 
204                 _have_valid_pieces = false;
205                 Changed (frequent);
206
207         } else if (
208                 property == SubtitleContentProperty::USE_SUBTITLES ||
209                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
210                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
211                 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
212                 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
213                 property == VideoContentProperty::VIDEO_CROP ||
214                 property == VideoContentProperty::VIDEO_SCALE ||
215                 property == VideoContentProperty::VIDEO_FRAME_RATE ||
216                 property == VideoContentProperty::VIDEO_FADE_IN ||
217                 property == VideoContentProperty::VIDEO_FADE_OUT
218                 ) {
219                 
220                 Changed (frequent);
221         }
222 }
223
224 void
225 Player::playlist_changed ()
226 {
227         _have_valid_pieces = false;
228         Changed (false);
229 }
230
231 void
232 Player::set_video_container_size (dcp::Size s)
233 {
234         _video_container_size = s;
235
236         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
237         _black_image->make_black ();
238 }
239
240 void
241 Player::film_changed (Film::Property p)
242 {
243         /* Here we should notice Film properties that affect our output, and
244            alert listeners that our output now would be different to how it was
245            last time we were run.
246         */
247
248         if (p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
249                 Changed (false);
250         }
251 }
252
253 list<PositionImage>
254 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
255 {
256         list<PositionImage> all;
257         
258         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
259                 if (!i->image) {
260                         continue;
261                 }
262
263                 /* We will scale the subtitle up to fit _video_container_size */
264                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
265                 
266                 /* Then we need a corrective translation, consisting of two parts:
267                  *
268                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
269                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
270                  *
271                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
272                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
273                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
274                  *
275                  * Combining these two translations gives these expressions.
276                  */
277
278                 all.push_back (
279                         PositionImage (
280                                 i->image->scale (
281                                         scaled_size,
282                                         dcp::YUV_TO_RGB_REC601,
283                                         i->image->pixel_format (),
284                                         true
285                                         ),
286                                 Position<int> (
287                                         rint (_video_container_size.width * i->rectangle.x),
288                                         rint (_video_container_size.height * i->rectangle.y)
289                                         )
290                                 )
291                         );
292         }
293
294         return all;
295 }
296
297 shared_ptr<PlayerVideo>
298 Player::black_player_video_frame (DCPTime time) const
299 {
300         return shared_ptr<PlayerVideo> (
301                 new PlayerVideo (
302                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
303                         time,
304                         Crop (),
305                         optional<float> (),
306                         _video_container_size,
307                         _video_container_size,
308                         EYES_BOTH,
309                         PART_WHOLE,
310                         PresetColourConversion::all().front().conversion
311                 )
312         );
313 }
314
315 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
316 list<shared_ptr<PlayerVideo> >
317 Player::get_video (DCPTime time, bool accurate)
318 {
319         if (!_have_valid_pieces) {
320                 setup_pieces ();
321         }
322
323         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
324                 time,
325                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
326                 );
327
328         list<shared_ptr<PlayerVideo> > pvf;
329
330         if (ov.empty ()) {
331                 /* No video content at this time */
332                 pvf.push_back (black_player_video_frame (time));
333         } else {
334                 /* Create a PlayerVideo from the content's video at this time */
335
336                 shared_ptr<Piece> piece = ov.back ();
337                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
338                 DCPOMATIC_ASSERT (decoder);
339                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
340                 DCPOMATIC_ASSERT (content);
341
342                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
343                 if (content_video.empty ()) {
344                         pvf.push_back (black_player_video_frame (time));
345                         return pvf;
346                 }
347                 
348                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
349
350                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
351                         pvf.push_back (
352                                 shared_ptr<PlayerVideo> (
353                                         new PlayerVideo (
354                                                 i->image,
355                                                 content_video_to_dcp (piece, i->frame),
356                                                 content->crop (),
357                                                 content->fade (i->frame),
358                                                 image_size,
359                                                 _video_container_size,
360                                                 i->eyes,
361                                                 i->part,
362                                                 content->colour_conversion ()
363                                                 )
364                                         )
365                                 );
366                 }
367         }
368
369         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
370
371         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
372
373         list<PositionImage> sub_images;
374
375         /* Image subtitles */
376         list<PositionImage> c = transform_image_subtitles (ps.image);
377         copy (c.begin(), c.end(), back_inserter (sub_images));
378
379         /* Text subtitles (rendered to an image) */
380         if (!ps.text.empty ()) {
381                 sub_images.push_back (render_subtitles (ps.text, _video_container_size));
382         }
383
384         if (!sub_images.empty ()) {
385                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
386                         (*i)->set_subtitle (merge (sub_images));
387                 }
388         }       
389                 
390         return pvf;
391 }
392
393 shared_ptr<AudioBuffers>
394 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
395 {
396         if (!_have_valid_pieces) {
397                 setup_pieces ();
398         }
399
400         Frame const length_frames = length.frames (_film->audio_frame_rate ());
401
402         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
403         audio->make_silent ();
404         
405         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
406         if (ov.empty ()) {
407                 return audio;
408         }
409
410         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
411
412                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
413                 DCPOMATIC_ASSERT (content);
414                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
415                 DCPOMATIC_ASSERT (decoder);
416
417                 /* The time that we should request from the content */
418                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
419                 Frame request_frames = length_frames;
420                 DCPTime offset;
421                 if (request < DCPTime ()) {
422                         /* We went off the start of the content, so we will need to offset
423                            the stuff we get back.
424                         */
425                         offset = -request;
426                         request_frames += request.frames (_film->audio_frame_rate ());
427                         if (request_frames < 0) {
428                                 request_frames = 0;
429                         }
430                         request = DCPTime ();
431                 }
432
433                 Frame const content_frame = dcp_to_content_audio (*i, request);
434
435                 BOOST_FOREACH (AudioStreamPtr j, content->audio_streams ()) {
436                         
437                         /* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
438                         ContentAudio all = decoder->get_audio (j, content_frame, request_frames, accurate);
439
440                         /* Gain */
441                         if (content->audio_gain() != 0) {
442                                 shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
443                                 gain->apply_gain (content->audio_gain ());
444                                 all.audio = gain;
445                         }
446
447                         /* Remap channels */
448                         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
449                         dcp_mapped->make_silent ();
450                         AudioMapping map = j->mapping ();
451                         for (int i = 0; i < map.content_channels(); ++i) {
452                                 for (int j = 0; j < _film->audio_channels(); ++j) {
453                                         if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
454                                                 dcp_mapped->accumulate_channel (
455                                                         all.audio.get(),
456                                                         i,
457                                                         j,
458                                                         map.get (i, static_cast<dcp::Channel> (j))
459                                                         );
460                                         }
461                                 }
462                         }
463                 
464                         all.audio = dcp_mapped;
465
466                         audio->accumulate_frames (
467                                 all.audio.get(),
468                                 content_frame - all.frame,
469                                 offset.frames (_film->audio_frame_rate()),
470                                 min (Frame (all.audio->frames()), request_frames)
471                                 );
472                 }
473         }
474
475         return audio;
476 }
477
478 Frame
479 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
480 {
481         /* s is the offset of t from the start position of this content */
482         DCPTime s = t - piece->content->position ();
483         s = DCPTime (max (DCPTime::Type (0), s.get ()));
484         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
485
486         /* Convert this to the content frame */
487         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) / piece->frc.factor ();
488 }
489
490 DCPTime
491 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
492 {
493         DCPTime t = DCPTime::from_frames (f * piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
494         if (t < DCPTime ()) {
495                 t = DCPTime ();
496         }
497
498         return t;
499 }
500
501 Frame
502 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
503 {
504         /* s is the offset of t from the start position of this content */
505         DCPTime s = t - piece->content->position ();
506         s = DCPTime (max (DCPTime::Type (0), s.get ()));
507         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
508
509         /* Convert this to the content frame */
510         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
511 }
512
513 ContentTime
514 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
515 {
516         /* s is the offset of t from the start position of this content */
517         DCPTime s = t - piece->content->position ();
518         s = DCPTime (max (DCPTime::Type (0), s.get ()));
519         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
520
521         return ContentTime (s + piece->content->trim_start(), piece->frc);
522 }
523
524 void
525 PlayerStatistics::dump (shared_ptr<Log> log) const
526 {
527         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
528         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
529 }
530
531 PlayerStatistics const &
532 Player::statistics () const
533 {
534         return _statistics;
535 }
536
537 PlayerSubtitles
538 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
539 {
540         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
541
542         PlayerSubtitles ps (time, length);
543
544         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
545                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
546                 if (!subtitle_content->use_subtitles ()) {
547                         continue;
548                 }
549
550                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
551                 ContentTime const from = dcp_to_content_subtitle (*j, time);
552                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
553                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
554
555                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
556                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
557                         
558                         /* Apply content's subtitle offsets */
559                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
560                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
561
562                         /* Apply content's subtitle scale */
563                         i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
564                         i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
565
566                         /* Apply a corrective translation to keep the subtitle centred after that scale */
567                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
568                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
569                         
570                         ps.image.push_back (i->sub);
571                 }
572
573                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
574                 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
575                         BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
576                                 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
577                                 s.set_size (s.size() * max (subtitle_content->subtitle_x_scale(), subtitle_content->subtitle_y_scale()));
578                                 ps.text.push_back (s);
579                         }
580                 }
581         }
582
583         return ps;
584 }
585
586 list<shared_ptr<Font> >
587 Player::get_subtitle_fonts ()
588 {
589         if (!_have_valid_pieces) {
590                 setup_pieces ();
591         }
592
593         list<shared_ptr<Font> > fonts;
594         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
595                 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
596                 if (sc) {
597                         /* XXX: things may go wrong if there are duplicate font IDs
598                            with different font files.
599                         */
600                         list<shared_ptr<Font> > f = sc->fonts ();
601                         copy (f.begin(), f.end(), back_inserter (fonts));
602                 }
603         }
604
605         return fonts;
606 }
607
608 /** Set this player never to produce any video data */
609 void
610 Player::set_ignore_video ()
611 {
612         _ignore_video = true;
613 }