Remove approximate size stuff where playback viewer would round
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include "player.h"
21 #include "film.h"
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
33 #include "playlist.h"
34 #include "job.h"
35 #include "image.h"
36 #include "raw_image_proxy.h"
37 #include "ratio.h"
38 #include "log.h"
39 #include "render_subtitles.h"
40 #include "config.h"
41 #include "content_video.h"
42 #include "player_video.h"
43 #include "frame_rate_change.h"
44 #include "dcp_content.h"
45 #include "dcp_decoder.h"
46 #include "dcp_subtitle_content.h"
47 #include "dcp_subtitle_decoder.h"
48 #include <boost/foreach.hpp>
49 #include <stdint.h>
50 #include <algorithm>
51
52 #include "i18n.h"
53
54 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
55
56 using std::list;
57 using std::cout;
58 using std::min;
59 using std::max;
60 using std::min;
61 using std::vector;
62 using std::pair;
63 using std::map;
64 using std::make_pair;
65 using boost::shared_ptr;
66 using boost::weak_ptr;
67 using boost::dynamic_pointer_cast;
68 using boost::optional;
69
70 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
71         : _film (f)
72         , _playlist (p)
73         , _have_valid_pieces (false)
74         , _ignore_video (false)
75 {
76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
79         set_video_container_size (_film->frame_size ());
80 }
81
82 void
83 Player::setup_pieces ()
84 {
85         list<shared_ptr<Piece> > old_pieces = _pieces;
86         _pieces.clear ();
87
88         ContentList content = _playlist->content ();
89
90         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
91
92                 if (!(*i)->paths_valid ()) {
93                         continue;
94                 }
95                 
96                 shared_ptr<Decoder> decoder;
97                 optional<FrameRateChange> frc;
98
99                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
100                 DCPTime best_overlap_t;
101                 shared_ptr<VideoContent> best_overlap;
102                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
103                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
104                         if (!vc) {
105                                 continue;
106                         }
107                         
108                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
109                         if (overlap > best_overlap_t) {
110                                 best_overlap = vc;
111                                 best_overlap_t = overlap;
112                         }
113                 }
114
115                 optional<FrameRateChange> best_overlap_frc;
116                 if (best_overlap) {
117                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
118                 } else {
119                         /* No video overlap; e.g. if the DCP is just audio */
120                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
121                 }
122
123                 /* FFmpeg */
124                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
125                 if (fc) {
126                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
127                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
128                 }
129
130                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
131                 if (dc) {
132                         decoder.reset (new DCPDecoder (dc));
133                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
134                 }
135
136                 /* ImageContent */
137                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
138                 if (ic) {
139                         /* See if we can re-use an old ImageDecoder */
140                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
141                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
142                                 if (imd && imd->content() == ic) {
143                                         decoder = imd;
144                                 }
145                         }
146
147                         if (!decoder) {
148                                 decoder.reset (new ImageDecoder (ic));
149                         }
150
151                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
152                 }
153
154                 /* SndfileContent */
155                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
156                 if (sc) {
157                         decoder.reset (new SndfileDecoder (sc));
158                         frc = best_overlap_frc;
159                 }
160
161                 /* SubRipContent */
162                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
163                 if (rc) {
164                         decoder.reset (new SubRipDecoder (rc));
165                         frc = best_overlap_frc;
166                 }
167
168                 /* DCPSubtitleContent */
169                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
170                 if (dsc) {
171                         decoder.reset (new DCPSubtitleDecoder (dsc));
172                         frc = best_overlap_frc;
173                 }
174
175                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
176                 if (vd && _ignore_video) {
177                         vd->set_ignore_video ();
178                 }
179
180                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
181         }
182
183         _have_valid_pieces = true;
184 }
185
186 void
187 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
188 {
189         shared_ptr<Content> c = w.lock ();
190         if (!c) {
191                 return;
192         }
193
194         if (
195                 property == ContentProperty::POSITION ||
196                 property == ContentProperty::LENGTH ||
197                 property == ContentProperty::TRIM_START ||
198                 property == ContentProperty::TRIM_END ||
199                 property == ContentProperty::PATH ||
200                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
201                 property == DCPContentProperty::CAN_BE_PLAYED
202                 ) {
203                 
204                 _have_valid_pieces = false;
205                 Changed (frequent);
206
207         } else if (
208                 property == SubtitleContentProperty::USE_SUBTITLES ||
209                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
210                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
211                 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
212                 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
213                 property == VideoContentProperty::VIDEO_CROP ||
214                 property == VideoContentProperty::VIDEO_SCALE ||
215                 property == VideoContentProperty::VIDEO_FRAME_RATE ||
216                 property == VideoContentProperty::VIDEO_FADE_IN ||
217                 property == VideoContentProperty::VIDEO_FADE_OUT
218                 ) {
219                 
220                 Changed (frequent);
221         }
222 }
223
224 void
225 Player::playlist_changed ()
226 {
227         _have_valid_pieces = false;
228         Changed (false);
229 }
230
231 void
232 Player::set_video_container_size (dcp::Size s)
233 {
234         _video_container_size = s;
235
236         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
237         _black_image->make_black ();
238 }
239
240 void
241 Player::film_changed (Film::Property p)
242 {
243         /* Here we should notice Film properties that affect our output, and
244            alert listeners that our output now would be different to how it was
245            last time we were run.
246         */
247
248         if (p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
249                 Changed (false);
250         }
251 }
252
253 list<PositionImage>
254 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
255 {
256         list<PositionImage> all;
257         
258         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
259                 if (!i->image) {
260                         continue;
261                 }
262
263                 /* We will scale the subtitle up to fit _video_container_size */
264                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
265                 
266                 /* Then we need a corrective translation, consisting of two parts:
267                  *
268                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
269                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
270                  *
271                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
272                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
273                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
274                  *
275                  * Combining these two translations gives these expressions.
276                  */
277
278                 all.push_back (
279                         PositionImage (
280                                 i->image->scale (
281                                         scaled_size,
282                                         dcp::YUV_TO_RGB_REC601,
283                                         i->image->pixel_format (),
284                                         true
285                                         ),
286                                 Position<int> (
287                                         rint (_video_container_size.width * i->rectangle.x),
288                                         rint (_video_container_size.height * i->rectangle.y)
289                                         )
290                                 )
291                         );
292         }
293
294         return all;
295 }
296
297 shared_ptr<PlayerVideo>
298 Player::black_player_video_frame (DCPTime time) const
299 {
300         return shared_ptr<PlayerVideo> (
301                 new PlayerVideo (
302                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
303                         time,
304                         Crop (),
305                         optional<float> (),
306                         _video_container_size,
307                         _video_container_size,
308                         EYES_BOTH,
309                         PART_WHOLE,
310                         Config::instance()->colour_conversions().front().conversion
311                 )
312         );
313 }
314
315 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
316 list<shared_ptr<PlayerVideo> >
317 Player::get_video (DCPTime time, bool accurate)
318 {
319         if (!_have_valid_pieces) {
320                 setup_pieces ();
321         }
322
323         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
324                 time,
325                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
326                 );
327
328         list<shared_ptr<PlayerVideo> > pvf;
329
330         if (ov.empty ()) {
331                 /* No video content at this time */
332                 pvf.push_back (black_player_video_frame (time));
333         } else {
334                 /* Create a PlayerVideo from the content's video at this time */
335
336                 shared_ptr<Piece> piece = ov.back ();
337                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
338                 DCPOMATIC_ASSERT (decoder);
339                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
340                 DCPOMATIC_ASSERT (content);
341
342                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
343                 if (content_video.empty ()) {
344                         pvf.push_back (black_player_video_frame (time));
345                         return pvf;
346                 }
347                 
348                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
349
350                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
351                         pvf.push_back (
352                                 shared_ptr<PlayerVideo> (
353                                         new PlayerVideo (
354                                                 i->image,
355                                                 content_video_to_dcp (piece, i->frame),
356                                                 content->crop (),
357                                                 content->fade (i->frame),
358                                                 image_size,
359                                                 _video_container_size,
360                                                 i->eyes,
361                                                 i->part,
362                                                 content->colour_conversion ()
363                                                 )
364                                         )
365                                 );
366                 }
367         }
368
369         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
370
371         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
372
373         list<PositionImage> sub_images;
374
375         /* Image subtitles */
376         list<PositionImage> c = transform_image_subtitles (ps.image);
377         copy (c.begin(), c.end(), back_inserter (sub_images));
378
379         /* Text subtitles (rendered to images) */
380         sub_images.push_back (render_subtitles (ps.text, _video_container_size));
381         
382         if (!sub_images.empty ()) {
383                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
384                         (*i)->set_subtitle (merge (sub_images));
385                 }
386         }       
387                 
388         return pvf;
389 }
390
391 shared_ptr<AudioBuffers>
392 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
393 {
394         if (!_have_valid_pieces) {
395                 setup_pieces ();
396         }
397
398         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
399
400         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
401         audio->make_silent ();
402         
403         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
404         if (ov.empty ()) {
405                 return audio;
406         }
407
408         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
409
410                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
411                 DCPOMATIC_ASSERT (content);
412                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
413                 DCPOMATIC_ASSERT (decoder);
414
415                 if (content->audio_frame_rate() == 0) {
416                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
417                          * audio stream).
418                          */
419                         continue;
420                 }
421
422                 /* The time that we should request from the content */
423                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
424                 AudioFrame request_frames = length_frames;
425                 DCPTime offset;
426                 if (request < DCPTime ()) {
427                         /* We went off the start of the content, so we will need to offset
428                            the stuff we get back.
429                         */
430                         offset = -request;
431                         request_frames += request.frames (_film->audio_frame_rate ());
432                         if (request_frames < 0) {
433                                 request_frames = 0;
434                         }
435                         request = DCPTime ();
436                 }
437
438                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
439
440                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
441                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, request_frames, accurate);
442
443                 /* Gain */
444                 if (content->audio_gain() != 0) {
445                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
446                         gain->apply_gain (content->audio_gain ());
447                         all->audio = gain;
448                 }
449
450                 /* Remap channels */
451                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
452                 dcp_mapped->make_silent ();
453                 AudioMapping map = content->audio_mapping ();
454                 for (int i = 0; i < map.content_channels(); ++i) {
455                         for (int j = 0; j < _film->audio_channels(); ++j) {
456                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
457                                         dcp_mapped->accumulate_channel (
458                                                 all->audio.get(),
459                                                 i,
460                                                 j,
461                                                 map.get (i, static_cast<dcp::Channel> (j))
462                                                 );
463                                 }
464                         }
465                 }
466                 
467                 all->audio = dcp_mapped;
468
469                 audio->accumulate_frames (
470                         all->audio.get(),
471                         content_frame - all->frame,
472                         offset.frames (_film->audio_frame_rate()),
473                         min (AudioFrame (all->audio->frames()), request_frames)
474                         );
475         }
476
477         return audio;
478 }
479
480 VideoFrame
481 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
482 {
483         /* s is the offset of t from the start position of this content */
484         DCPTime s = t - piece->content->position ();
485         s = DCPTime (max (DCPTime::Type (0), s.get ()));
486         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
487
488         /* Convert this to the content frame */
489         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) / piece->frc.factor ();
490 }
491
492 DCPTime
493 Player::content_video_to_dcp (shared_ptr<const Piece> piece, VideoFrame f) const
494 {
495         DCPTime t = DCPTime::from_frames (f * piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
496         if (t < DCPTime ()) {
497                 t = DCPTime ();
498         }
499
500         return t;
501 }
502
503 AudioFrame
504 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
505 {
506         /* s is the offset of t from the start position of this content */
507         DCPTime s = t - piece->content->position ();
508         s = DCPTime (max (DCPTime::Type (0), s.get ()));
509         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
510
511         /* Convert this to the content frame */
512         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
513 }
514
515 ContentTime
516 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
517 {
518         /* s is the offset of t from the start position of this content */
519         DCPTime s = t - piece->content->position ();
520         s = DCPTime (max (DCPTime::Type (0), s.get ()));
521         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
522
523         return ContentTime (s + piece->content->trim_start(), piece->frc);
524 }
525
526 void
527 PlayerStatistics::dump (shared_ptr<Log> log) const
528 {
529         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
530         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
531 }
532
533 PlayerStatistics const &
534 Player::statistics () const
535 {
536         return _statistics;
537 }
538
539 PlayerSubtitles
540 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
541 {
542         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
543
544         PlayerSubtitles ps (time, length);
545
546         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
547                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
548                 if (!subtitle_content->use_subtitles ()) {
549                         continue;
550                 }
551
552                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
553                 ContentTime const from = dcp_to_content_subtitle (*j, time);
554                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
555                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
556
557                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
558                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
559                         
560                         /* Apply content's subtitle offsets */
561                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
562                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
563
564                         /* Apply content's subtitle scale */
565                         i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
566                         i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
567
568                         /* Apply a corrective translation to keep the subtitle centred after that scale */
569                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
570                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
571                         
572                         ps.image.push_back (i->sub);
573                 }
574
575                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
576                 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
577                         BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
578                                 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
579                                 s.set_size (s.size() * max (subtitle_content->subtitle_x_scale(), subtitle_content->subtitle_y_scale()));
580                                 ps.text.push_back (s);
581                         }
582                 }
583         }
584
585         return ps;
586 }
587
588 list<shared_ptr<Font> >
589 Player::get_subtitle_fonts ()
590 {
591         if (!_have_valid_pieces) {
592                 setup_pieces ();
593         }
594
595         list<shared_ptr<Font> > fonts;
596         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
597                 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
598                 if (sc) {
599                         /* XXX: things may go wrong if there are duplicate font IDs
600                            with different font files.
601                         */
602                         list<shared_ptr<Font> > f = sc->fonts ();
603                         copy (f.begin(), f.end(), back_inserter (fonts));
604                 }
605         }
606
607         return fonts;
608 }
609
610 /** Set this player never to produce any video data */
611 void
612 Player::set_ignore_video ()
613 {
614         _ignore_video = true;
615 }