Remove Scaler config and use SWS_BICUBIC everywhere.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include "player.h"
21 #include "film.h"
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
33 #include "playlist.h"
34 #include "job.h"
35 #include "image.h"
36 #include "raw_image_proxy.h"
37 #include "ratio.h"
38 #include "log.h"
39 #include "render_subtitles.h"
40 #include "config.h"
41 #include "content_video.h"
42 #include "player_video.h"
43 #include "frame_rate_change.h"
44 #include "dcp_content.h"
45 #include "dcp_decoder.h"
46 #include "dcp_subtitle_content.h"
47 #include "dcp_subtitle_decoder.h"
48 #include <boost/foreach.hpp>
49 #include <stdint.h>
50 #include <algorithm>
51
52 #include "i18n.h"
53
54 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
55
56 using std::list;
57 using std::cout;
58 using std::min;
59 using std::max;
60 using std::min;
61 using std::vector;
62 using std::pair;
63 using std::map;
64 using std::make_pair;
65 using boost::shared_ptr;
66 using boost::weak_ptr;
67 using boost::dynamic_pointer_cast;
68 using boost::optional;
69
70 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
71         : _film (f)
72         , _playlist (p)
73         , _have_valid_pieces (false)
74         , _approximate_size (false)
75         , _ignore_video (false)
76 {
77         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
78         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
79         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
80         set_video_container_size (_film->frame_size ());
81 }
82
83 void
84 Player::setup_pieces ()
85 {
86         list<shared_ptr<Piece> > old_pieces = _pieces;
87         _pieces.clear ();
88
89         ContentList content = _playlist->content ();
90
91         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
92
93                 if (!(*i)->paths_valid ()) {
94                         continue;
95                 }
96                 
97                 shared_ptr<Decoder> decoder;
98                 optional<FrameRateChange> frc;
99
100                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
101                 DCPTime best_overlap_t;
102                 shared_ptr<VideoContent> best_overlap;
103                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
104                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
105                         if (!vc) {
106                                 continue;
107                         }
108                         
109                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
110                         if (overlap > best_overlap_t) {
111                                 best_overlap = vc;
112                                 best_overlap_t = overlap;
113                         }
114                 }
115
116                 optional<FrameRateChange> best_overlap_frc;
117                 if (best_overlap) {
118                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
119                 } else {
120                         /* No video overlap; e.g. if the DCP is just audio */
121                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
122                 }
123
124                 /* FFmpeg */
125                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
126                 if (fc) {
127                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
128                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
129                 }
130
131                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
132                 if (dc) {
133                         decoder.reset (new DCPDecoder (dc));
134                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
135                 }
136
137                 /* ImageContent */
138                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
139                 if (ic) {
140                         /* See if we can re-use an old ImageDecoder */
141                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
142                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
143                                 if (imd && imd->content() == ic) {
144                                         decoder = imd;
145                                 }
146                         }
147
148                         if (!decoder) {
149                                 decoder.reset (new ImageDecoder (ic));
150                         }
151
152                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
153                 }
154
155                 /* SndfileContent */
156                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
157                 if (sc) {
158                         decoder.reset (new SndfileDecoder (sc));
159                         frc = best_overlap_frc;
160                 }
161
162                 /* SubRipContent */
163                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
164                 if (rc) {
165                         decoder.reset (new SubRipDecoder (rc));
166                         frc = best_overlap_frc;
167                 }
168
169                 /* DCPSubtitleContent */
170                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
171                 if (dsc) {
172                         decoder.reset (new DCPSubtitleDecoder (dsc));
173                         frc = best_overlap_frc;
174                 }
175
176                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
177                 if (vd && _ignore_video) {
178                         vd->set_ignore_video ();
179                 }
180
181                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
182         }
183
184         _have_valid_pieces = true;
185 }
186
187 void
188 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
189 {
190         shared_ptr<Content> c = w.lock ();
191         if (!c) {
192                 return;
193         }
194
195         if (
196                 property == ContentProperty::POSITION ||
197                 property == ContentProperty::LENGTH ||
198                 property == ContentProperty::TRIM_START ||
199                 property == ContentProperty::TRIM_END ||
200                 property == ContentProperty::PATH ||
201                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
202                 property == DCPContentProperty::CAN_BE_PLAYED
203                 ) {
204                 
205                 _have_valid_pieces = false;
206                 Changed (frequent);
207
208         } else if (
209                 property == SubtitleContentProperty::USE_SUBTITLES ||
210                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
211                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
212                 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
213                 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
214                 property == VideoContentProperty::VIDEO_CROP ||
215                 property == VideoContentProperty::VIDEO_SCALE ||
216                 property == VideoContentProperty::VIDEO_FRAME_RATE ||
217                 property == VideoContentProperty::VIDEO_FADE_IN ||
218                 property == VideoContentProperty::VIDEO_FADE_OUT
219                 ) {
220                 
221                 Changed (frequent);
222         }
223 }
224
225 void
226 Player::playlist_changed ()
227 {
228         _have_valid_pieces = false;
229         Changed (false);
230 }
231
232 void
233 Player::set_video_container_size (dcp::Size s)
234 {
235         _video_container_size = s;
236
237         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
238         _black_image->make_black ();
239 }
240
241 void
242 Player::film_changed (Film::Property p)
243 {
244         /* Here we should notice Film properties that affect our output, and
245            alert listeners that our output now would be different to how it was
246            last time we were run.
247         */
248
249         if (p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
250                 Changed (false);
251         }
252 }
253
254 list<PositionImage>
255 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
256 {
257         list<PositionImage> all;
258         
259         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
260                 if (!i->image) {
261                         continue;
262                 }
263
264                 /* We will scale the subtitle up to fit _video_container_size */
265                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
266                 
267                 /* Then we need a corrective translation, consisting of two parts:
268                  *
269                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
270                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
271                  *
272                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
273                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
274                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
275                  *
276                  * Combining these two translations gives these expressions.
277                  */
278
279                 all.push_back (
280                         PositionImage (
281                                 i->image->scale (
282                                         scaled_size,
283                                         i->image->pixel_format (),
284                                         true
285                                         ),
286                                 Position<int> (
287                                         rint (_video_container_size.width * i->rectangle.x),
288                                         rint (_video_container_size.height * i->rectangle.y)
289                                         )
290                                 )
291                         );
292         }
293
294         return all;
295 }
296
297 void
298 Player::set_approximate_size ()
299 {
300         _approximate_size = true;
301 }
302
303 shared_ptr<PlayerVideo>
304 Player::black_player_video_frame (DCPTime time) const
305 {
306         return shared_ptr<PlayerVideo> (
307                 new PlayerVideo (
308                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
309                         time,
310                         Crop (),
311                         optional<float> (),
312                         _video_container_size,
313                         _video_container_size,
314                         EYES_BOTH,
315                         PART_WHOLE,
316                         Config::instance()->colour_conversions().front().conversion
317                 )
318         );
319 }
320
321 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
322 list<shared_ptr<PlayerVideo> >
323 Player::get_video (DCPTime time, bool accurate)
324 {
325         if (!_have_valid_pieces) {
326                 setup_pieces ();
327         }
328
329         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
330                 time,
331                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
332                 );
333
334         list<shared_ptr<PlayerVideo> > pvf;
335
336         if (ov.empty ()) {
337                 /* No video content at this time */
338                 pvf.push_back (black_player_video_frame (time));
339         } else {
340                 /* Create a PlayerVideo from the content's video at this time */
341
342                 shared_ptr<Piece> piece = ov.back ();
343                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
344                 DCPOMATIC_ASSERT (decoder);
345                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
346                 DCPOMATIC_ASSERT (content);
347
348                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
349                 if (content_video.empty ()) {
350                         pvf.push_back (black_player_video_frame (time));
351                         return pvf;
352                 }
353                 
354                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size (), _approximate_size ? 4 : 1);
355
356                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
357                         pvf.push_back (
358                                 shared_ptr<PlayerVideo> (
359                                         new PlayerVideo (
360                                                 i->image,
361                                                 content_video_to_dcp (piece, i->frame),
362                                                 content->crop (),
363                                                 content->fade (i->frame),
364                                                 image_size,
365                                                 _video_container_size,
366                                                 i->eyes,
367                                                 i->part,
368                                                 content->colour_conversion ()
369                                                 )
370                                         )
371                                 );
372                 }
373         }
374
375         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
376
377         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
378
379         list<PositionImage> sub_images;
380
381         /* Image subtitles */
382         list<PositionImage> c = transform_image_subtitles (ps.image);
383         copy (c.begin(), c.end(), back_inserter (sub_images));
384
385         /* Text subtitles (rendered to images) */
386         sub_images.push_back (render_subtitles (ps.text, _video_container_size));
387         
388         if (!sub_images.empty ()) {
389                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
390                         (*i)->set_subtitle (merge (sub_images));
391                 }
392         }       
393                 
394         return pvf;
395 }
396
397 shared_ptr<AudioBuffers>
398 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
399 {
400         if (!_have_valid_pieces) {
401                 setup_pieces ();
402         }
403
404         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
405
406         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
407         audio->make_silent ();
408         
409         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
410         if (ov.empty ()) {
411                 return audio;
412         }
413
414         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
415
416                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
417                 DCPOMATIC_ASSERT (content);
418                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
419                 DCPOMATIC_ASSERT (decoder);
420
421                 if (content->audio_frame_rate() == 0) {
422                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
423                          * audio stream).
424                          */
425                         continue;
426                 }
427
428                 /* The time that we should request from the content */
429                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
430                 AudioFrame request_frames = length_frames;
431                 DCPTime offset;
432                 if (request < DCPTime ()) {
433                         /* We went off the start of the content, so we will need to offset
434                            the stuff we get back.
435                         */
436                         offset = -request;
437                         request_frames += request.frames (_film->audio_frame_rate ());
438                         if (request_frames < 0) {
439                                 request_frames = 0;
440                         }
441                         request = DCPTime ();
442                 }
443
444                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
445
446                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
447                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, request_frames, accurate);
448
449                 /* Gain */
450                 if (content->audio_gain() != 0) {
451                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
452                         gain->apply_gain (content->audio_gain ());
453                         all->audio = gain;
454                 }
455
456                 /* Remap channels */
457                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
458                 dcp_mapped->make_silent ();
459                 AudioMapping map = content->audio_mapping ();
460                 for (int i = 0; i < map.content_channels(); ++i) {
461                         for (int j = 0; j < _film->audio_channels(); ++j) {
462                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
463                                         dcp_mapped->accumulate_channel (
464                                                 all->audio.get(),
465                                                 i,
466                                                 j,
467                                                 map.get (i, static_cast<dcp::Channel> (j))
468                                                 );
469                                 }
470                         }
471                 }
472                 
473                 all->audio = dcp_mapped;
474
475                 audio->accumulate_frames (
476                         all->audio.get(),
477                         content_frame - all->frame,
478                         offset.frames (_film->audio_frame_rate()),
479                         min (AudioFrame (all->audio->frames()), request_frames)
480                         );
481         }
482
483         return audio;
484 }
485
486 VideoFrame
487 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
488 {
489         /* s is the offset of t from the start position of this content */
490         DCPTime s = t - piece->content->position ();
491         s = DCPTime (max (DCPTime::Type (0), s.get ()));
492         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
493
494         /* Convert this to the content frame */
495         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) / piece->frc.factor ();
496 }
497
498 DCPTime
499 Player::content_video_to_dcp (shared_ptr<const Piece> piece, VideoFrame f) const
500 {
501         DCPTime t = DCPTime::from_frames (f * piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
502         if (t < DCPTime ()) {
503                 t = DCPTime ();
504         }
505
506         return t;
507 }
508
509 AudioFrame
510 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
511 {
512         /* s is the offset of t from the start position of this content */
513         DCPTime s = t - piece->content->position ();
514         s = DCPTime (max (DCPTime::Type (0), s.get ()));
515         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
516
517         /* Convert this to the content frame */
518         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
519 }
520
521 ContentTime
522 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
523 {
524         /* s is the offset of t from the start position of this content */
525         DCPTime s = t - piece->content->position ();
526         s = DCPTime (max (DCPTime::Type (0), s.get ()));
527         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
528
529         return ContentTime (s + piece->content->trim_start(), piece->frc);
530 }
531
532 void
533 PlayerStatistics::dump (shared_ptr<Log> log) const
534 {
535         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
536         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
537 }
538
539 PlayerStatistics const &
540 Player::statistics () const
541 {
542         return _statistics;
543 }
544
545 PlayerSubtitles
546 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
547 {
548         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
549
550         PlayerSubtitles ps (time, length);
551
552         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
553                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
554                 if (!subtitle_content->use_subtitles ()) {
555                         continue;
556                 }
557
558                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
559                 ContentTime const from = dcp_to_content_subtitle (*j, time);
560                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
561                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
562
563                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
564                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
565                         
566                         /* Apply content's subtitle offsets */
567                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
568                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
569
570                         /* Apply content's subtitle scale */
571                         i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
572                         i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
573
574                         /* Apply a corrective translation to keep the subtitle centred after that scale */
575                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
576                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
577                         
578                         ps.image.push_back (i->sub);
579                 }
580
581                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
582                 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
583                         BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
584                                 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
585                                 s.set_size (s.size() * max (subtitle_content->subtitle_x_scale(), subtitle_content->subtitle_y_scale()));
586                                 ps.text.push_back (s);
587                         }
588                 }
589         }
590
591         return ps;
592 }
593
594 list<shared_ptr<Font> >
595 Player::get_subtitle_fonts ()
596 {
597         if (!_have_valid_pieces) {
598                 setup_pieces ();
599         }
600
601         list<shared_ptr<Font> > fonts;
602         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
603                 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
604                 if (sc) {
605                         /* XXX: things may go wrong if there are duplicate font IDs
606                            with different font files.
607                         */
608                         list<shared_ptr<Font> > f = sc->fonts ();
609                         copy (f.begin(), f.end(), back_inserter (fonts));
610                 }
611         }
612
613         return fonts;
614 }
615
616 /** Set this player never to produce any video data */
617 void
618 Player::set_ignore_video ()
619 {
620         _ignore_video = true;
621 }