I can't see the point of rounding the image size again after calling
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include "player.h"
21 #include "film.h"
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
33 #include "playlist.h"
34 #include "job.h"
35 #include "image.h"
36 #include "raw_image_proxy.h"
37 #include "ratio.h"
38 #include "log.h"
39 #include "scaler.h"
40 #include "render_subtitles.h"
41 #include "config.h"
42 #include "content_video.h"
43 #include "player_video.h"
44 #include "frame_rate_change.h"
45 #include "dcp_content.h"
46 #include "dcp_decoder.h"
47 #include "dcp_subtitle_content.h"
48 #include "dcp_subtitle_decoder.h"
49 #include <boost/foreach.hpp>
50 #include <stdint.h>
51 #include <algorithm>
52
53 #include "i18n.h"
54
55 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
56
57 using std::list;
58 using std::cout;
59 using std::min;
60 using std::max;
61 using std::min;
62 using std::vector;
63 using std::pair;
64 using std::map;
65 using std::make_pair;
66 using boost::shared_ptr;
67 using boost::weak_ptr;
68 using boost::dynamic_pointer_cast;
69 using boost::optional;
70
71 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
72         : _film (f)
73         , _playlist (p)
74         , _have_valid_pieces (false)
75         , _approximate_size (false)
76         , _ignore_video (false)
77 {
78         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
79         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
80         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
81         set_video_container_size (_film->frame_size ());
82 }
83
84 void
85 Player::setup_pieces ()
86 {
87         list<shared_ptr<Piece> > old_pieces = _pieces;
88         _pieces.clear ();
89
90         ContentList content = _playlist->content ();
91
92         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
93
94                 if (!(*i)->paths_valid ()) {
95                         continue;
96                 }
97                 
98                 shared_ptr<Decoder> decoder;
99                 optional<FrameRateChange> frc;
100
101                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
102                 DCPTime best_overlap_t;
103                 shared_ptr<VideoContent> best_overlap;
104                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
105                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
106                         if (!vc) {
107                                 continue;
108                         }
109                         
110                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
111                         if (overlap > best_overlap_t) {
112                                 best_overlap = vc;
113                                 best_overlap_t = overlap;
114                         }
115                 }
116
117                 optional<FrameRateChange> best_overlap_frc;
118                 if (best_overlap) {
119                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
120                 } else {
121                         /* No video overlap; e.g. if the DCP is just audio */
122                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
123                 }
124
125                 /* FFmpeg */
126                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
127                 if (fc) {
128                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
129                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
130                 }
131
132                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
133                 if (dc) {
134                         decoder.reset (new DCPDecoder (dc));
135                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
136                 }
137
138                 /* ImageContent */
139                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
140                 if (ic) {
141                         /* See if we can re-use an old ImageDecoder */
142                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
143                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
144                                 if (imd && imd->content() == ic) {
145                                         decoder = imd;
146                                 }
147                         }
148
149                         if (!decoder) {
150                                 decoder.reset (new ImageDecoder (ic));
151                         }
152
153                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
154                 }
155
156                 /* SndfileContent */
157                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
158                 if (sc) {
159                         decoder.reset (new SndfileDecoder (sc));
160                         frc = best_overlap_frc;
161                 }
162
163                 /* SubRipContent */
164                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
165                 if (rc) {
166                         decoder.reset (new SubRipDecoder (rc));
167                         frc = best_overlap_frc;
168                 }
169
170                 /* DCPSubtitleContent */
171                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
172                 if (dsc) {
173                         decoder.reset (new DCPSubtitleDecoder (dsc));
174                         frc = best_overlap_frc;
175                 }
176
177                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
178                 if (vd && _ignore_video) {
179                         vd->set_ignore_video ();
180                 }
181
182                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
183         }
184
185         _have_valid_pieces = true;
186 }
187
188 void
189 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
190 {
191         shared_ptr<Content> c = w.lock ();
192         if (!c) {
193                 return;
194         }
195
196         if (
197                 property == ContentProperty::POSITION ||
198                 property == ContentProperty::LENGTH ||
199                 property == ContentProperty::TRIM_START ||
200                 property == ContentProperty::TRIM_END ||
201                 property == ContentProperty::PATH ||
202                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
203                 property == DCPContentProperty::CAN_BE_PLAYED
204                 ) {
205                 
206                 _have_valid_pieces = false;
207                 Changed (frequent);
208
209         } else if (
210                 property == SubtitleContentProperty::USE_SUBTITLES ||
211                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
212                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
213                 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
214                 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
215                 property == VideoContentProperty::VIDEO_CROP ||
216                 property == VideoContentProperty::VIDEO_SCALE ||
217                 property == VideoContentProperty::VIDEO_FRAME_RATE ||
218                 property == VideoContentProperty::VIDEO_FADE_IN ||
219                 property == VideoContentProperty::VIDEO_FADE_OUT
220                 ) {
221                 
222                 Changed (frequent);
223         }
224 }
225
226 /** @param already_resampled true if this data has already been through the chain up to the resampler */
227 void
228 Player::playlist_changed ()
229 {
230         _have_valid_pieces = false;
231         Changed (false);
232 }
233
234 void
235 Player::set_video_container_size (dcp::Size s)
236 {
237         _video_container_size = s;
238
239         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
240         _black_image->make_black ();
241 }
242
243 void
244 Player::film_changed (Film::Property p)
245 {
246         /* Here we should notice Film properties that affect our output, and
247            alert listeners that our output now would be different to how it was
248            last time we were run.
249         */
250
251         if (p == Film::SCALER || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
252                 Changed (false);
253         }
254 }
255
256 list<PositionImage>
257 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
258 {
259         list<PositionImage> all;
260         
261         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
262                 if (!i->image) {
263                         continue;
264                 }
265
266                 /* We will scale the subtitle up to fit _video_container_size */
267                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
268                 
269                 /* Then we need a corrective translation, consisting of two parts:
270                  *
271                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
272                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
273                  *
274                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
275                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
276                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
277                  *
278                  * Combining these two translations gives these expressions.
279                  */
280
281                 all.push_back (
282                         PositionImage (
283                                 i->image->scale (
284                                         scaled_size,
285                                         Scaler::from_id ("bicubic"),
286                                         i->image->pixel_format (),
287                                         true
288                                         ),
289                                 Position<int> (
290                                         rint (_video_container_size.width * i->rectangle.x),
291                                         rint (_video_container_size.height * i->rectangle.y)
292                                         )
293                                 )
294                         );
295         }
296
297         return all;
298 }
299
300 void
301 Player::set_approximate_size ()
302 {
303         _approximate_size = true;
304 }
305
306 shared_ptr<PlayerVideo>
307 Player::black_player_video_frame (DCPTime time) const
308 {
309         return shared_ptr<PlayerVideo> (
310                 new PlayerVideo (
311                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
312                         time,
313                         Crop (),
314                         optional<float> (),
315                         _video_container_size,
316                         _video_container_size,
317                         Scaler::from_id ("bicubic"),
318                         EYES_BOTH,
319                         PART_WHOLE,
320                         Config::instance()->colour_conversions().front().conversion
321                 )
322         );
323 }
324
325 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
326 list<shared_ptr<PlayerVideo> >
327 Player::get_video (DCPTime time, bool accurate)
328 {
329         if (!_have_valid_pieces) {
330                 setup_pieces ();
331         }
332
333         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
334                 time,
335                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
336                 );
337
338         list<shared_ptr<PlayerVideo> > pvf;
339
340         if (ov.empty ()) {
341                 /* No video content at this time */
342                 pvf.push_back (black_player_video_frame (time));
343         } else {
344                 /* Create a PlayerVideo from the content's video at this time */
345
346                 shared_ptr<Piece> piece = ov.back ();
347                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
348                 DCPOMATIC_ASSERT (decoder);
349                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
350                 DCPOMATIC_ASSERT (content);
351
352                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
353                 if (content_video.empty ()) {
354                         pvf.push_back (black_player_video_frame (time));
355                         return pvf;
356                 }
357                 
358                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size (), _approximate_size ? 4 : 1);
359
360                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
361                         pvf.push_back (
362                                 shared_ptr<PlayerVideo> (
363                                         new PlayerVideo (
364                                                 i->image,
365                                                 content_video_to_dcp (piece, i->frame),
366                                                 content->crop (),
367                                                 content->fade (i->frame),
368                                                 image_size,
369                                                 _video_container_size,
370                                                 _film->scaler(),
371                                                 i->eyes,
372                                                 i->part,
373                                                 content->colour_conversion ()
374                                                 )
375                                         )
376                                 );
377                 }
378         }
379
380         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
381
382         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
383
384         list<PositionImage> sub_images;
385
386         /* Image subtitles */
387         list<PositionImage> c = transform_image_subtitles (ps.image);
388         copy (c.begin(), c.end(), back_inserter (sub_images));
389
390         /* Text subtitles (rendered to images) */
391         sub_images.push_back (render_subtitles (ps.text, _video_container_size));
392         
393         if (!sub_images.empty ()) {
394                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
395                         (*i)->set_subtitle (merge (sub_images));
396                 }
397         }       
398                 
399         return pvf;
400 }
401
402 shared_ptr<AudioBuffers>
403 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
404 {
405         if (!_have_valid_pieces) {
406                 setup_pieces ();
407         }
408
409         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
410
411         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
412         audio->make_silent ();
413         
414         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
415         if (ov.empty ()) {
416                 return audio;
417         }
418
419         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
420
421                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
422                 DCPOMATIC_ASSERT (content);
423                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
424                 DCPOMATIC_ASSERT (decoder);
425
426                 if (content->audio_frame_rate() == 0) {
427                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
428                          * audio stream).
429                          */
430                         continue;
431                 }
432
433                 /* The time that we should request from the content */
434                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
435                 AudioFrame request_frames = length_frames;
436                 DCPTime offset;
437                 if (request < DCPTime ()) {
438                         /* We went off the start of the content, so we will need to offset
439                            the stuff we get back.
440                         */
441                         offset = -request;
442                         request_frames += request.frames (_film->audio_frame_rate ());
443                         if (request_frames < 0) {
444                                 request_frames = 0;
445                         }
446                         request = DCPTime ();
447                 }
448
449                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
450
451                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
452                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, request_frames, accurate);
453
454                 /* Gain */
455                 if (content->audio_gain() != 0) {
456                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
457                         gain->apply_gain (content->audio_gain ());
458                         all->audio = gain;
459                 }
460
461                 /* Remap channels */
462                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
463                 dcp_mapped->make_silent ();
464                 AudioMapping map = content->audio_mapping ();
465                 for (int i = 0; i < map.content_channels(); ++i) {
466                         for (int j = 0; j < _film->audio_channels(); ++j) {
467                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
468                                         dcp_mapped->accumulate_channel (
469                                                 all->audio.get(),
470                                                 i,
471                                                 j,
472                                                 map.get (i, static_cast<dcp::Channel> (j))
473                                                 );
474                                 }
475                         }
476                 }
477                 
478                 all->audio = dcp_mapped;
479
480                 audio->accumulate_frames (
481                         all->audio.get(),
482                         content_frame - all->frame,
483                         offset.frames (_film->audio_frame_rate()),
484                         min (AudioFrame (all->audio->frames()), request_frames)
485                         );
486         }
487
488         return audio;
489 }
490
491 VideoFrame
492 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
493 {
494         /* s is the offset of t from the start position of this content */
495         DCPTime s = t - piece->content->position ();
496         s = DCPTime (max (DCPTime::Type (0), s.get ()));
497         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
498
499         /* Convert this to the content frame */
500         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) / piece->frc.factor ();
501 }
502
503 DCPTime
504 Player::content_video_to_dcp (shared_ptr<const Piece> piece, VideoFrame f) const
505 {
506         DCPTime t = DCPTime::from_frames (f * piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
507         if (t < DCPTime ()) {
508                 t = DCPTime ();
509         }
510
511         return t;
512 }
513
514 AudioFrame
515 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
516 {
517         /* s is the offset of t from the start position of this content */
518         DCPTime s = t - piece->content->position ();
519         s = DCPTime (max (DCPTime::Type (0), s.get ()));
520         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
521
522         /* Convert this to the content frame */
523         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
524 }
525
526 ContentTime
527 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
528 {
529         /* s is the offset of t from the start position of this content */
530         DCPTime s = t - piece->content->position ();
531         s = DCPTime (max (DCPTime::Type (0), s.get ()));
532         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
533
534         return ContentTime (s + piece->content->trim_start(), piece->frc);
535 }
536
537 void
538 PlayerStatistics::dump (shared_ptr<Log> log) const
539 {
540         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
541         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
542 }
543
544 PlayerStatistics const &
545 Player::statistics () const
546 {
547         return _statistics;
548 }
549
550 PlayerSubtitles
551 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
552 {
553         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
554
555         PlayerSubtitles ps (time, length);
556
557         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
558                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
559                 if (!subtitle_content->use_subtitles ()) {
560                         continue;
561                 }
562
563                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
564                 ContentTime const from = dcp_to_content_subtitle (*j, time);
565                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
566                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
567
568                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
569                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
570                         
571                         /* Apply content's subtitle offsets */
572                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
573                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
574
575                         /* Apply content's subtitle scale */
576                         i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
577                         i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
578
579                         /* Apply a corrective translation to keep the subtitle centred after that scale */
580                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
581                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
582                         
583                         ps.image.push_back (i->sub);
584                 }
585
586                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
587                 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
588                         BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
589                                 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
590                                 s.set_size (s.size() * max (subtitle_content->subtitle_x_scale(), subtitle_content->subtitle_y_scale()));
591                                 ps.text.push_back (s);
592                         }
593                 }
594         }
595
596         return ps;
597 }
598
599 list<shared_ptr<Font> >
600 Player::get_subtitle_fonts ()
601 {
602         if (!_have_valid_pieces) {
603                 setup_pieces ();
604         }
605
606         list<shared_ptr<Font> > fonts;
607         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
608                 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
609                 if (sc) {
610                         /* XXX: things may go wrong if there are duplicate font IDs
611                            with different font files.
612                         */
613                         list<shared_ptr<Font> > f = sc->fonts ();
614                         copy (f.begin(), f.end(), back_inserter (fonts));
615                 }
616         }
617
618         return fonts;
619 }
620
621 /** Set this player never to produce any video data */
622 void
623 Player::set_ignore_video ()
624 {
625         _ignore_video = true;
626 }