Fix update of preview when video fade changes.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include "player.h"
21 #include "film.h"
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
33 #include "playlist.h"
34 #include "job.h"
35 #include "image.h"
36 #include "raw_image_proxy.h"
37 #include "ratio.h"
38 #include "log.h"
39 #include "scaler.h"
40 #include "render_subtitles.h"
41 #include "config.h"
42 #include "content_video.h"
43 #include "player_video.h"
44 #include "frame_rate_change.h"
45 #include "dcp_content.h"
46 #include "dcp_decoder.h"
47 #include "dcp_subtitle_content.h"
48 #include "dcp_subtitle_decoder.h"
49 #include <boost/foreach.hpp>
50 #include <stdint.h>
51 #include <algorithm>
52
53 #include "i18n.h"
54
55 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
56
57 using std::list;
58 using std::cout;
59 using std::min;
60 using std::max;
61 using std::min;
62 using std::vector;
63 using std::pair;
64 using std::map;
65 using std::make_pair;
66 using boost::shared_ptr;
67 using boost::weak_ptr;
68 using boost::dynamic_pointer_cast;
69 using boost::optional;
70
71 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
72         : _film (f)
73         , _playlist (p)
74         , _have_valid_pieces (false)
75         , _approximate_size (false)
76         , _ignore_video (false)
77 {
78         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
79         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
80         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
81         set_video_container_size (_film->frame_size ());
82 }
83
84 void
85 Player::setup_pieces ()
86 {
87         list<shared_ptr<Piece> > old_pieces = _pieces;
88         _pieces.clear ();
89
90         ContentList content = _playlist->content ();
91
92         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
93
94                 if (!(*i)->paths_valid ()) {
95                         continue;
96                 }
97                 
98                 shared_ptr<Decoder> decoder;
99                 optional<FrameRateChange> frc;
100
101                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
102                 DCPTime best_overlap_t;
103                 shared_ptr<VideoContent> best_overlap;
104                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
105                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
106                         if (!vc) {
107                                 continue;
108                         }
109                         
110                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
111                         if (overlap > best_overlap_t) {
112                                 best_overlap = vc;
113                                 best_overlap_t = overlap;
114                         }
115                 }
116
117                 optional<FrameRateChange> best_overlap_frc;
118                 if (best_overlap) {
119                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
120                 } else {
121                         /* No video overlap; e.g. if the DCP is just audio */
122                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
123                 }
124
125                 /* FFmpeg */
126                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
127                 if (fc) {
128                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
129                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
130                 }
131
132                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
133                 if (dc) {
134                         decoder.reset (new DCPDecoder (dc));
135                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
136                 }
137
138                 /* ImageContent */
139                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
140                 if (ic) {
141                         /* See if we can re-use an old ImageDecoder */
142                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
143                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
144                                 if (imd && imd->content() == ic) {
145                                         decoder = imd;
146                                 }
147                         }
148
149                         if (!decoder) {
150                                 decoder.reset (new ImageDecoder (ic));
151                         }
152
153                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
154                 }
155
156                 /* SndfileContent */
157                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
158                 if (sc) {
159                         decoder.reset (new SndfileDecoder (sc));
160                         frc = best_overlap_frc;
161                 }
162
163                 /* SubRipContent */
164                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
165                 if (rc) {
166                         decoder.reset (new SubRipDecoder (rc));
167                         frc = best_overlap_frc;
168                 }
169
170                 /* DCPSubtitleContent */
171                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
172                 if (dsc) {
173                         decoder.reset (new DCPSubtitleDecoder (dsc));
174                         frc = best_overlap_frc;
175                 }
176
177                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
178                 if (vd && _ignore_video) {
179                         vd->set_ignore_video ();
180                 }
181
182                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
183         }
184
185         _have_valid_pieces = true;
186 }
187
188 void
189 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
190 {
191         shared_ptr<Content> c = w.lock ();
192         if (!c) {
193                 return;
194         }
195
196         if (
197                 property == ContentProperty::POSITION ||
198                 property == ContentProperty::LENGTH ||
199                 property == ContentProperty::TRIM_START ||
200                 property == ContentProperty::TRIM_END ||
201                 property == ContentProperty::PATH ||
202                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
203                 property == DCPContentProperty::CAN_BE_PLAYED
204                 ) {
205                 
206                 _have_valid_pieces = false;
207                 Changed (frequent);
208
209         } else if (
210                 property == SubtitleContentProperty::USE_SUBTITLES ||
211                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
212                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
213                 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
214                 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
215                 property == VideoContentProperty::VIDEO_CROP ||
216                 property == VideoContentProperty::VIDEO_SCALE ||
217                 property == VideoContentProperty::VIDEO_FRAME_RATE ||
218                 property == VideoContentProperty::VIDEO_FADE_IN ||
219                 property == VideoContentProperty::VIDEO_FADE_OUT
220                 ) {
221                 
222                 Changed (frequent);
223         }
224 }
225
226 /** @param already_resampled true if this data has already been through the chain up to the resampler */
227 void
228 Player::playlist_changed ()
229 {
230         _have_valid_pieces = false;
231         Changed (false);
232 }
233
234 void
235 Player::set_video_container_size (dcp::Size s)
236 {
237         _video_container_size = s;
238
239         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
240         _black_image->make_black ();
241 }
242
243 void
244 Player::film_changed (Film::Property p)
245 {
246         /* Here we should notice Film properties that affect our output, and
247            alert listeners that our output now would be different to how it was
248            last time we were run.
249         */
250
251         if (p == Film::SCALER || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
252                 Changed (false);
253         }
254 }
255
256 list<PositionImage>
257 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
258 {
259         list<PositionImage> all;
260         
261         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
262                 if (!i->image) {
263                         continue;
264                 }
265
266                 /* We will scale the subtitle up to fit _video_container_size */
267                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
268                 
269                 /* Then we need a corrective translation, consisting of two parts:
270                  *
271                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
272                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
273                  *
274                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
275                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
276                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
277                  *
278                  * Combining these two translations gives these expressions.
279                  */
280
281                 all.push_back (
282                         PositionImage (
283                                 i->image->scale (
284                                         scaled_size,
285                                         Scaler::from_id ("bicubic"),
286                                         i->image->pixel_format (),
287                                         true
288                                         ),
289                                 Position<int> (
290                                         rint (_video_container_size.width * i->rectangle.x),
291                                         rint (_video_container_size.height * i->rectangle.y)
292                                         )
293                                 )
294                         );
295         }
296
297         return all;
298 }
299
300 void
301 Player::set_approximate_size ()
302 {
303         _approximate_size = true;
304 }
305
306 shared_ptr<PlayerVideo>
307 Player::black_player_video_frame (DCPTime time) const
308 {
309         return shared_ptr<PlayerVideo> (
310                 new PlayerVideo (
311                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
312                         time,
313                         Crop (),
314                         optional<float> (),
315                         _video_container_size,
316                         _video_container_size,
317                         Scaler::from_id ("bicubic"),
318                         EYES_BOTH,
319                         PART_WHOLE,
320                         Config::instance()->colour_conversions().front().conversion
321                 )
322         );
323 }
324
325 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
326 list<shared_ptr<PlayerVideo> >
327 Player::get_video (DCPTime time, bool accurate)
328 {
329         if (!_have_valid_pieces) {
330                 setup_pieces ();
331         }
332
333         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
334                 time,
335                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
336                 );
337
338         list<shared_ptr<PlayerVideo> > pvf;
339
340         if (ov.empty ()) {
341                 /* No video content at this time */
342                 pvf.push_back (black_player_video_frame (time));
343         } else {
344                 /* Create a PlayerVideo from the content's video at this time */
345
346                 shared_ptr<Piece> piece = ov.back ();
347                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
348                 DCPOMATIC_ASSERT (decoder);
349                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
350                 DCPOMATIC_ASSERT (content);
351
352                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
353                 if (content_video.empty ()) {
354                         pvf.push_back (black_player_video_frame (time));
355                         return pvf;
356                 }
357                 
358                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size (), _approximate_size ? 4 : 1);
359                 if (_approximate_size) {
360                         image_size.width &= ~3;
361                         image_size.height &= ~3;
362                 }
363
364                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
365                         pvf.push_back (
366                                 shared_ptr<PlayerVideo> (
367                                         new PlayerVideo (
368                                                 i->image,
369                                                 content_video_to_dcp (piece, i->frame),
370                                                 content->crop (),
371                                                 content->fade (i->frame),
372                                                 image_size,
373                                                 _video_container_size,
374                                                 _film->scaler(),
375                                                 i->eyes,
376                                                 i->part,
377                                                 content->colour_conversion ()
378                                                 )
379                                         )
380                                 );
381                 }
382         }
383
384         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
385
386         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
387
388         list<PositionImage> sub_images;
389
390         /* Image subtitles */
391         list<PositionImage> c = transform_image_subtitles (ps.image);
392         copy (c.begin(), c.end(), back_inserter (sub_images));
393
394         /* Text subtitles (rendered to images) */
395         sub_images.push_back (render_subtitles (ps.text, _video_container_size));
396         
397         if (!sub_images.empty ()) {
398                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
399                         (*i)->set_subtitle (merge (sub_images));
400                 }
401         }       
402                 
403         return pvf;
404 }
405
406 shared_ptr<AudioBuffers>
407 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
408 {
409         if (!_have_valid_pieces) {
410                 setup_pieces ();
411         }
412
413         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
414
415         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
416         audio->make_silent ();
417         
418         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
419         if (ov.empty ()) {
420                 return audio;
421         }
422
423         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
424
425                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
426                 DCPOMATIC_ASSERT (content);
427                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
428                 DCPOMATIC_ASSERT (decoder);
429
430                 if (content->audio_frame_rate() == 0) {
431                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
432                          * audio stream).
433                          */
434                         continue;
435                 }
436
437                 /* The time that we should request from the content */
438                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
439                 DCPTime offset;
440                 if (request < DCPTime ()) {
441                         /* We went off the start of the content, so we will need to offset
442                            the stuff we get back.
443                         */
444                         offset = -request;
445                         request = DCPTime ();
446                 }
447
448                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
449
450                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
451                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, length_frames, accurate);
452
453                 /* Gain */
454                 if (content->audio_gain() != 0) {
455                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
456                         gain->apply_gain (content->audio_gain ());
457                         all->audio = gain;
458                 }
459
460                 /* Remap channels */
461                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
462                 dcp_mapped->make_silent ();
463                 AudioMapping map = content->audio_mapping ();
464                 for (int i = 0; i < map.content_channels(); ++i) {
465                         for (int j = 0; j < _film->audio_channels(); ++j) {
466                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
467                                         dcp_mapped->accumulate_channel (
468                                                 all->audio.get(),
469                                                 i,
470                                                 j,
471                                                 map.get (i, static_cast<dcp::Channel> (j))
472                                                 );
473                                 }
474                         }
475                 }
476                 
477                 all->audio = dcp_mapped;
478
479                 audio->accumulate_frames (
480                         all->audio.get(),
481                         content_frame - all->frame,
482                         offset.frames (_film->audio_frame_rate()),
483                         min (AudioFrame (all->audio->frames()), length_frames) - offset.frames (_film->audio_frame_rate ())
484                         );
485         }
486
487         return audio;
488 }
489
490 VideoFrame
491 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
492 {
493         /* s is the offset of t from the start position of this content */
494         DCPTime s = t - piece->content->position ();
495         s = DCPTime (max (DCPTime::Type (0), s.get ()));
496         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
497
498         /* Convert this to the content frame */
499         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) / piece->frc.factor ();
500 }
501
502 DCPTime
503 Player::content_video_to_dcp (shared_ptr<const Piece> piece, VideoFrame f) const
504 {
505         DCPTime t = DCPTime::from_frames (f * piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
506         if (t < DCPTime ()) {
507                 t = DCPTime ();
508         }
509
510         return t;
511 }
512
513 AudioFrame
514 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
515 {
516         /* s is the offset of t from the start position of this content */
517         DCPTime s = t - piece->content->position ();
518         s = DCPTime (max (DCPTime::Type (0), s.get ()));
519         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
520
521         /* Convert this to the content frame */
522         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
523 }
524
525 ContentTime
526 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
527 {
528         /* s is the offset of t from the start position of this content */
529         DCPTime s = t - piece->content->position ();
530         s = DCPTime (max (DCPTime::Type (0), s.get ()));
531         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
532
533         return ContentTime (s + piece->content->trim_start(), piece->frc);
534 }
535
536 void
537 PlayerStatistics::dump (shared_ptr<Log> log) const
538 {
539         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
540         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
541 }
542
543 PlayerStatistics const &
544 Player::statistics () const
545 {
546         return _statistics;
547 }
548
549 PlayerSubtitles
550 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
551 {
552         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
553
554         PlayerSubtitles ps (time, length);
555
556         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
557                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
558                 if (!subtitle_content->use_subtitles ()) {
559                         continue;
560                 }
561
562                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
563                 ContentTime const from = dcp_to_content_subtitle (*j, time);
564                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
565                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
566
567                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
568                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
569                         
570                         /* Apply content's subtitle offsets */
571                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
572                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
573
574                         /* Apply content's subtitle scale */
575                         i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
576                         i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
577
578                         /* Apply a corrective translation to keep the subtitle centred after that scale */
579                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
580                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
581                         
582                         ps.image.push_back (i->sub);
583                 }
584
585                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
586                 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
587                         BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
588                                 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
589                                 s.set_size (s.size() * max (subtitle_content->subtitle_x_scale(), subtitle_content->subtitle_y_scale()));
590                                 ps.text.push_back (s);
591                         }
592                 }
593         }
594
595         return ps;
596 }
597
598 list<shared_ptr<Font> >
599 Player::get_subtitle_fonts ()
600 {
601         if (!_have_valid_pieces) {
602                 setup_pieces ();
603         }
604
605         list<shared_ptr<Font> > fonts;
606         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
607                 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
608                 if (sc) {
609                         /* XXX: things may go wrong if there are duplicate font IDs
610                            with different font files.
611                         */
612                         list<shared_ptr<Font> > f = sc->fonts ();
613                         copy (f.begin(), f.end(), back_inserter (fonts));
614                 }
615         }
616
617         return fonts;
618 }
619
620 /** Set this player never to produce any video data */
621 void
622 Player::set_ignore_video ()
623 {
624         _ignore_video = true;
625 }