If we are requesting audio from before the start of a piece of content we need to...
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include "player.h"
21 #include "film.h"
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
33 #include "playlist.h"
34 #include "job.h"
35 #include "image.h"
36 #include "raw_image_proxy.h"
37 #include "ratio.h"
38 #include "log.h"
39 #include "scaler.h"
40 #include "render_subtitles.h"
41 #include "config.h"
42 #include "content_video.h"
43 #include "player_video.h"
44 #include "frame_rate_change.h"
45 #include "dcp_content.h"
46 #include "dcp_decoder.h"
47 #include "dcp_subtitle_content.h"
48 #include "dcp_subtitle_decoder.h"
49 #include <boost/foreach.hpp>
50 #include <stdint.h>
51 #include <algorithm>
52
53 #include "i18n.h"
54
55 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
56
57 using std::list;
58 using std::cout;
59 using std::min;
60 using std::max;
61 using std::min;
62 using std::vector;
63 using std::pair;
64 using std::map;
65 using std::make_pair;
66 using boost::shared_ptr;
67 using boost::weak_ptr;
68 using boost::dynamic_pointer_cast;
69 using boost::optional;
70
71 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
72         : _film (f)
73         , _playlist (p)
74         , _have_valid_pieces (false)
75         , _approximate_size (false)
76         , _ignore_video (false)
77 {
78         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
79         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
80         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
81         set_video_container_size (_film->frame_size ());
82 }
83
84 void
85 Player::setup_pieces ()
86 {
87         list<shared_ptr<Piece> > old_pieces = _pieces;
88         _pieces.clear ();
89
90         ContentList content = _playlist->content ();
91
92         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
93
94                 if (!(*i)->paths_valid ()) {
95                         continue;
96                 }
97                 
98                 shared_ptr<Decoder> decoder;
99                 optional<FrameRateChange> frc;
100
101                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
102                 DCPTime best_overlap_t;
103                 shared_ptr<VideoContent> best_overlap;
104                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
105                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
106                         if (!vc) {
107                                 continue;
108                         }
109                         
110                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
111                         if (overlap > best_overlap_t) {
112                                 best_overlap = vc;
113                                 best_overlap_t = overlap;
114                         }
115                 }
116
117                 optional<FrameRateChange> best_overlap_frc;
118                 if (best_overlap) {
119                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
120                 } else {
121                         /* No video overlap; e.g. if the DCP is just audio */
122                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
123                 }
124
125                 /* FFmpeg */
126                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
127                 if (fc) {
128                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
129                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
130                 }
131
132                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
133                 if (dc) {
134                         decoder.reset (new DCPDecoder (dc));
135                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
136                 }
137
138                 /* ImageContent */
139                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
140                 if (ic) {
141                         /* See if we can re-use an old ImageDecoder */
142                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
143                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
144                                 if (imd && imd->content() == ic) {
145                                         decoder = imd;
146                                 }
147                         }
148
149                         if (!decoder) {
150                                 decoder.reset (new ImageDecoder (ic));
151                         }
152
153                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
154                 }
155
156                 /* SndfileContent */
157                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
158                 if (sc) {
159                         decoder.reset (new SndfileDecoder (sc));
160                         frc = best_overlap_frc;
161                 }
162
163                 /* SubRipContent */
164                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
165                 if (rc) {
166                         decoder.reset (new SubRipDecoder (rc));
167                         frc = best_overlap_frc;
168                 }
169
170                 /* DCPSubtitleContent */
171                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
172                 if (dsc) {
173                         decoder.reset (new DCPSubtitleDecoder (dsc));
174                         frc = best_overlap_frc;
175                 }
176
177                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
178                 if (vd && _ignore_video) {
179                         vd->set_ignore_video ();
180                 }
181
182                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
183         }
184
185         _have_valid_pieces = true;
186 }
187
188 void
189 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
190 {
191         shared_ptr<Content> c = w.lock ();
192         if (!c) {
193                 return;
194         }
195
196         if (
197                 property == ContentProperty::POSITION ||
198                 property == ContentProperty::LENGTH ||
199                 property == ContentProperty::TRIM_START ||
200                 property == ContentProperty::TRIM_END ||
201                 property == ContentProperty::PATH ||
202                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
203                 property == DCPContentProperty::CAN_BE_PLAYED
204                 ) {
205                 
206                 _have_valid_pieces = false;
207                 Changed (frequent);
208
209         } else if (
210                 property == SubtitleContentProperty::USE_SUBTITLES ||
211                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
212                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
213                 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
214                 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
215                 property == VideoContentProperty::VIDEO_CROP ||
216                 property == VideoContentProperty::VIDEO_SCALE ||
217                 property == VideoContentProperty::VIDEO_FRAME_RATE ||
218                 property == VideoContentProperty::VIDEO_FADE_IN ||
219                 property == VideoContentProperty::VIDEO_FADE_OUT
220                 ) {
221                 
222                 Changed (frequent);
223         }
224 }
225
226 /** @param already_resampled true if this data has already been through the chain up to the resampler */
227 void
228 Player::playlist_changed ()
229 {
230         _have_valid_pieces = false;
231         Changed (false);
232 }
233
234 void
235 Player::set_video_container_size (dcp::Size s)
236 {
237         _video_container_size = s;
238
239         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
240         _black_image->make_black ();
241 }
242
243 void
244 Player::film_changed (Film::Property p)
245 {
246         /* Here we should notice Film properties that affect our output, and
247            alert listeners that our output now would be different to how it was
248            last time we were run.
249         */
250
251         if (p == Film::SCALER || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
252                 Changed (false);
253         }
254 }
255
256 list<PositionImage>
257 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
258 {
259         list<PositionImage> all;
260         
261         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
262                 if (!i->image) {
263                         continue;
264                 }
265
266                 /* We will scale the subtitle up to fit _video_container_size */
267                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
268                 
269                 /* Then we need a corrective translation, consisting of two parts:
270                  *
271                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
272                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
273                  *
274                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
275                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
276                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
277                  *
278                  * Combining these two translations gives these expressions.
279                  */
280
281                 all.push_back (
282                         PositionImage (
283                                 i->image->scale (
284                                         scaled_size,
285                                         Scaler::from_id ("bicubic"),
286                                         i->image->pixel_format (),
287                                         true
288                                         ),
289                                 Position<int> (
290                                         rint (_video_container_size.width * i->rectangle.x),
291                                         rint (_video_container_size.height * i->rectangle.y)
292                                         )
293                                 )
294                         );
295         }
296
297         return all;
298 }
299
300 void
301 Player::set_approximate_size ()
302 {
303         _approximate_size = true;
304 }
305
306 shared_ptr<PlayerVideo>
307 Player::black_player_video_frame (DCPTime time) const
308 {
309         return shared_ptr<PlayerVideo> (
310                 new PlayerVideo (
311                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
312                         time,
313                         Crop (),
314                         optional<float> (),
315                         _video_container_size,
316                         _video_container_size,
317                         Scaler::from_id ("bicubic"),
318                         EYES_BOTH,
319                         PART_WHOLE,
320                         Config::instance()->colour_conversions().front().conversion
321                 )
322         );
323 }
324
325 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
326 list<shared_ptr<PlayerVideo> >
327 Player::get_video (DCPTime time, bool accurate)
328 {
329         if (!_have_valid_pieces) {
330                 setup_pieces ();
331         }
332
333         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
334                 time,
335                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
336                 );
337
338         list<shared_ptr<PlayerVideo> > pvf;
339
340         if (ov.empty ()) {
341                 /* No video content at this time */
342                 pvf.push_back (black_player_video_frame (time));
343         } else {
344                 /* Create a PlayerVideo from the content's video at this time */
345
346                 shared_ptr<Piece> piece = ov.back ();
347                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
348                 DCPOMATIC_ASSERT (decoder);
349                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
350                 DCPOMATIC_ASSERT (content);
351
352                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
353                 if (content_video.empty ()) {
354                         pvf.push_back (black_player_video_frame (time));
355                         return pvf;
356                 }
357                 
358                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size (), _approximate_size ? 4 : 1);
359                 if (_approximate_size) {
360                         image_size.width &= ~3;
361                         image_size.height &= ~3;
362                 }
363
364                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
365                         pvf.push_back (
366                                 shared_ptr<PlayerVideo> (
367                                         new PlayerVideo (
368                                                 i->image,
369                                                 content_video_to_dcp (piece, i->frame),
370                                                 content->crop (),
371                                                 content->fade (i->frame),
372                                                 image_size,
373                                                 _video_container_size,
374                                                 _film->scaler(),
375                                                 i->eyes,
376                                                 i->part,
377                                                 content->colour_conversion ()
378                                                 )
379                                         )
380                                 );
381                 }
382         }
383
384         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
385
386         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
387
388         list<PositionImage> sub_images;
389
390         /* Image subtitles */
391         list<PositionImage> c = transform_image_subtitles (ps.image);
392         copy (c.begin(), c.end(), back_inserter (sub_images));
393
394         /* Text subtitles (rendered to images) */
395         sub_images.push_back (render_subtitles (ps.text, _video_container_size));
396         
397         if (!sub_images.empty ()) {
398                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
399                         (*i)->set_subtitle (merge (sub_images));
400                 }
401         }       
402                 
403         return pvf;
404 }
405
406 shared_ptr<AudioBuffers>
407 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
408 {
409         if (!_have_valid_pieces) {
410                 setup_pieces ();
411         }
412
413         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
414
415         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
416         audio->make_silent ();
417         
418         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
419         if (ov.empty ()) {
420                 return audio;
421         }
422
423         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
424
425                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
426                 DCPOMATIC_ASSERT (content);
427                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
428                 DCPOMATIC_ASSERT (decoder);
429
430                 if (content->audio_frame_rate() == 0) {
431                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
432                          * audio stream).
433                          */
434                         continue;
435                 }
436
437                 /* The time that we should request from the content */
438                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
439                 AudioFrame request_frames = length_frames;
440                 DCPTime offset;
441                 if (request < DCPTime ()) {
442                         /* We went off the start of the content, so we will need to offset
443                            the stuff we get back.
444                         */
445                         offset = -request;
446                         request_frames += request.frames (_film->audio_frame_rate ());
447                         if (request_frames < 0) {
448                                 request_frames = 0;
449                         }
450                         request = DCPTime ();
451                 }
452
453                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
454
455                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
456                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, request_frames, accurate);
457
458                 /* Gain */
459                 if (content->audio_gain() != 0) {
460                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
461                         gain->apply_gain (content->audio_gain ());
462                         all->audio = gain;
463                 }
464
465                 /* Remap channels */
466                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
467                 dcp_mapped->make_silent ();
468                 AudioMapping map = content->audio_mapping ();
469                 for (int i = 0; i < map.content_channels(); ++i) {
470                         for (int j = 0; j < _film->audio_channels(); ++j) {
471                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
472                                         dcp_mapped->accumulate_channel (
473                                                 all->audio.get(),
474                                                 i,
475                                                 j,
476                                                 map.get (i, static_cast<dcp::Channel> (j))
477                                                 );
478                                 }
479                         }
480                 }
481                 
482                 all->audio = dcp_mapped;
483
484                 audio->accumulate_frames (
485                         all->audio.get(),
486                         content_frame - all->frame,
487                         offset.frames (_film->audio_frame_rate()),
488                         min (AudioFrame (all->audio->frames()), request_frames)
489                         );
490         }
491
492         return audio;
493 }
494
495 VideoFrame
496 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
497 {
498         /* s is the offset of t from the start position of this content */
499         DCPTime s = t - piece->content->position ();
500         s = DCPTime (max (DCPTime::Type (0), s.get ()));
501         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
502
503         /* Convert this to the content frame */
504         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) / piece->frc.factor ();
505 }
506
507 DCPTime
508 Player::content_video_to_dcp (shared_ptr<const Piece> piece, VideoFrame f) const
509 {
510         DCPTime t = DCPTime::from_frames (f * piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
511         if (t < DCPTime ()) {
512                 t = DCPTime ();
513         }
514
515         return t;
516 }
517
518 AudioFrame
519 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
520 {
521         /* s is the offset of t from the start position of this content */
522         DCPTime s = t - piece->content->position ();
523         s = DCPTime (max (DCPTime::Type (0), s.get ()));
524         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
525
526         /* Convert this to the content frame */
527         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
528 }
529
530 ContentTime
531 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
532 {
533         /* s is the offset of t from the start position of this content */
534         DCPTime s = t - piece->content->position ();
535         s = DCPTime (max (DCPTime::Type (0), s.get ()));
536         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
537
538         return ContentTime (s + piece->content->trim_start(), piece->frc);
539 }
540
541 void
542 PlayerStatistics::dump (shared_ptr<Log> log) const
543 {
544         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
545         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
546 }
547
548 PlayerStatistics const &
549 Player::statistics () const
550 {
551         return _statistics;
552 }
553
554 PlayerSubtitles
555 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
556 {
557         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
558
559         PlayerSubtitles ps (time, length);
560
561         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
562                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
563                 if (!subtitle_content->use_subtitles ()) {
564                         continue;
565                 }
566
567                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
568                 ContentTime const from = dcp_to_content_subtitle (*j, time);
569                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
570                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
571
572                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
573                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
574                         
575                         /* Apply content's subtitle offsets */
576                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
577                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
578
579                         /* Apply content's subtitle scale */
580                         i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
581                         i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
582
583                         /* Apply a corrective translation to keep the subtitle centred after that scale */
584                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
585                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
586                         
587                         ps.image.push_back (i->sub);
588                 }
589
590                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
591                 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
592                         BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
593                                 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
594                                 s.set_size (s.size() * max (subtitle_content->subtitle_x_scale(), subtitle_content->subtitle_y_scale()));
595                                 ps.text.push_back (s);
596                         }
597                 }
598         }
599
600         return ps;
601 }
602
603 list<shared_ptr<Font> >
604 Player::get_subtitle_fonts ()
605 {
606         if (!_have_valid_pieces) {
607                 setup_pieces ();
608         }
609
610         list<shared_ptr<Font> > fonts;
611         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
612                 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
613                 if (sc) {
614                         /* XXX: things may go wrong if there are duplicate font IDs
615                            with different font files.
616                         */
617                         list<shared_ptr<Font> > f = sc->fonts ();
618                         copy (f.begin(), f.end(), back_inserter (fonts));
619                 }
620         }
621
622         return fonts;
623 }
624
625 /** Set this player never to produce any video data */
626 void
627 Player::set_ignore_video ()
628 {
629         _ignore_video = true;
630 }