Assorted image subtitle fixes.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include "player.h"
21 #include "film.h"
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
33 #include "playlist.h"
34 #include "job.h"
35 #include "image.h"
36 #include "raw_image_proxy.h"
37 #include "ratio.h"
38 #include "log.h"
39 #include "render_subtitles.h"
40 #include "config.h"
41 #include "content_video.h"
42 #include "player_video.h"
43 #include "frame_rate_change.h"
44 #include "dcp_content.h"
45 #include "dcp_decoder.h"
46 #include "dcp_subtitle_content.h"
47 #include "dcp_subtitle_decoder.h"
48 #include <boost/foreach.hpp>
49 #include <stdint.h>
50 #include <algorithm>
51
52 #include "i18n.h"
53
54 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
55
56 using std::list;
57 using std::cout;
58 using std::min;
59 using std::max;
60 using std::min;
61 using std::vector;
62 using std::pair;
63 using std::map;
64 using std::make_pair;
65 using boost::shared_ptr;
66 using boost::weak_ptr;
67 using boost::dynamic_pointer_cast;
68 using boost::optional;
69
70 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
71         : _film (f)
72         , _playlist (p)
73         , _have_valid_pieces (false)
74         , _ignore_video (false)
75 {
76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
79         set_video_container_size (_film->frame_size ());
80 }
81
82 void
83 Player::setup_pieces ()
84 {
85         list<shared_ptr<Piece> > old_pieces = _pieces;
86         _pieces.clear ();
87
88         ContentList content = _playlist->content ();
89
90         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
91
92                 if (!(*i)->paths_valid ()) {
93                         continue;
94                 }
95                 
96                 shared_ptr<Decoder> decoder;
97                 optional<FrameRateChange> frc;
98
99                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
100                 DCPTime best_overlap_t;
101                 shared_ptr<VideoContent> best_overlap;
102                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
103                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
104                         if (!vc) {
105                                 continue;
106                         }
107                         
108                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
109                         if (overlap > best_overlap_t) {
110                                 best_overlap = vc;
111                                 best_overlap_t = overlap;
112                         }
113                 }
114
115                 optional<FrameRateChange> best_overlap_frc;
116                 if (best_overlap) {
117                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
118                 } else {
119                         /* No video overlap; e.g. if the DCP is just audio */
120                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
121                 }
122
123                 /* FFmpeg */
124                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
125                 if (fc) {
126                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
127                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
128                 }
129
130                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
131                 if (dc) {
132                         decoder.reset (new DCPDecoder (dc));
133                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
134                 }
135
136                 /* ImageContent */
137                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
138                 if (ic) {
139                         /* See if we can re-use an old ImageDecoder */
140                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
141                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
142                                 if (imd && imd->content() == ic) {
143                                         decoder = imd;
144                                 }
145                         }
146
147                         if (!decoder) {
148                                 decoder.reset (new ImageDecoder (ic));
149                         }
150
151                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
152                 }
153
154                 /* SndfileContent */
155                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
156                 if (sc) {
157                         decoder.reset (new SndfileDecoder (sc));
158                         frc = best_overlap_frc;
159                 }
160
161                 /* SubRipContent */
162                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
163                 if (rc) {
164                         decoder.reset (new SubRipDecoder (rc));
165                         frc = best_overlap_frc;
166                 }
167
168                 /* DCPSubtitleContent */
169                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
170                 if (dsc) {
171                         decoder.reset (new DCPSubtitleDecoder (dsc));
172                         frc = best_overlap_frc;
173                 }
174
175                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
176                 if (vd && _ignore_video) {
177                         vd->set_ignore_video ();
178                 }
179
180                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
181         }
182
183         _have_valid_pieces = true;
184 }
185
186 void
187 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
188 {
189         shared_ptr<Content> c = w.lock ();
190         if (!c) {
191                 return;
192         }
193
194         if (
195                 property == ContentProperty::POSITION ||
196                 property == ContentProperty::LENGTH ||
197                 property == ContentProperty::TRIM_START ||
198                 property == ContentProperty::TRIM_END ||
199                 property == ContentProperty::PATH ||
200                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
201                 property == DCPContentProperty::CAN_BE_PLAYED
202                 ) {
203                 
204                 _have_valid_pieces = false;
205                 Changed (frequent);
206
207         } else if (
208                 property == SubtitleContentProperty::USE_SUBTITLES ||
209                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
210                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
211                 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
212                 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
213                 property == VideoContentProperty::VIDEO_CROP ||
214                 property == VideoContentProperty::VIDEO_SCALE ||
215                 property == VideoContentProperty::VIDEO_FRAME_RATE ||
216                 property == VideoContentProperty::VIDEO_FADE_IN ||
217                 property == VideoContentProperty::VIDEO_FADE_OUT
218                 ) {
219                 
220                 Changed (frequent);
221         }
222 }
223
224 void
225 Player::playlist_changed ()
226 {
227         _have_valid_pieces = false;
228         Changed (false);
229 }
230
231 void
232 Player::set_video_container_size (dcp::Size s)
233 {
234         _video_container_size = s;
235
236         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
237         _black_image->make_black ();
238 }
239
240 void
241 Player::film_changed (Film::Property p)
242 {
243         /* Here we should notice Film properties that affect our output, and
244            alert listeners that our output now would be different to how it was
245            last time we were run.
246         */
247
248         if (p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
249                 Changed (false);
250         }
251 }
252
253 list<PositionImage>
254 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
255 {
256         list<PositionImage> all;
257         
258         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
259                 if (!i->image) {
260                         continue;
261                 }
262
263                 /* We will scale the subtitle up to fit _video_container_size */
264                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
265                 
266                 /* Then we need a corrective translation, consisting of two parts:
267                  *
268                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
269                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
270                  *
271                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
272                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
273                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
274                  *
275                  * Combining these two translations gives these expressions.
276                  */
277
278                 all.push_back (
279                         PositionImage (
280                                 i->image->scale (
281                                         scaled_size,
282                                         dcp::YUV_TO_RGB_REC601,
283                                         i->image->pixel_format (),
284                                         true
285                                         ),
286                                 Position<int> (
287                                         rint (_video_container_size.width * i->rectangle.x),
288                                         rint (_video_container_size.height * i->rectangle.y)
289                                         )
290                                 )
291                         );
292         }
293
294         return all;
295 }
296
297 shared_ptr<PlayerVideo>
298 Player::black_player_video_frame (DCPTime time) const
299 {
300         return shared_ptr<PlayerVideo> (
301                 new PlayerVideo (
302                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
303                         time,
304                         Crop (),
305                         optional<float> (),
306                         _video_container_size,
307                         _video_container_size,
308                         EYES_BOTH,
309                         PART_WHOLE,
310                         Config::instance()->colour_conversions().front().conversion
311                 )
312         );
313 }
314
315 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
316 list<shared_ptr<PlayerVideo> >
317 Player::get_video (DCPTime time, bool accurate)
318 {
319         if (!_have_valid_pieces) {
320                 setup_pieces ();
321         }
322
323         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
324                 time,
325                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
326                 );
327
328         list<shared_ptr<PlayerVideo> > pvf;
329
330         if (ov.empty ()) {
331                 /* No video content at this time */
332                 pvf.push_back (black_player_video_frame (time));
333         } else {
334                 /* Create a PlayerVideo from the content's video at this time */
335
336                 shared_ptr<Piece> piece = ov.back ();
337                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
338                 DCPOMATIC_ASSERT (decoder);
339                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
340                 DCPOMATIC_ASSERT (content);
341
342                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
343                 if (content_video.empty ()) {
344                         pvf.push_back (black_player_video_frame (time));
345                         return pvf;
346                 }
347                 
348                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
349
350                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
351                         pvf.push_back (
352                                 shared_ptr<PlayerVideo> (
353                                         new PlayerVideo (
354                                                 i->image,
355                                                 content_video_to_dcp (piece, i->frame),
356                                                 content->crop (),
357                                                 content->fade (i->frame),
358                                                 image_size,
359                                                 _video_container_size,
360                                                 i->eyes,
361                                                 i->part,
362                                                 content->colour_conversion ()
363                                                 )
364                                         )
365                                 );
366                 }
367         }
368
369         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
370
371         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
372
373         list<PositionImage> sub_images;
374
375         /* Image subtitles */
376         list<PositionImage> c = transform_image_subtitles (ps.image);
377         copy (c.begin(), c.end(), back_inserter (sub_images));
378
379         /* Text subtitles (rendered to an image) */
380         if (!ps.text.empty ()) {
381                 sub_images.push_back (render_subtitles (ps.text, _video_container_size));
382         }
383
384         if (!sub_images.empty ()) {
385                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
386                         (*i)->set_subtitle (merge (sub_images));
387                 }
388         }       
389                 
390         return pvf;
391 }
392
393 shared_ptr<AudioBuffers>
394 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
395 {
396         if (!_have_valid_pieces) {
397                 setup_pieces ();
398         }
399
400         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
401
402         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
403         audio->make_silent ();
404         
405         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
406         if (ov.empty ()) {
407                 return audio;
408         }
409
410         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
411
412                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
413                 DCPOMATIC_ASSERT (content);
414                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
415                 DCPOMATIC_ASSERT (decoder);
416
417                 if (content->audio_frame_rate() == 0) {
418                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
419                          * audio stream).
420                          */
421                         continue;
422                 }
423
424                 /* The time that we should request from the content */
425                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
426                 AudioFrame request_frames = length_frames;
427                 DCPTime offset;
428                 if (request < DCPTime ()) {
429                         /* We went off the start of the content, so we will need to offset
430                            the stuff we get back.
431                         */
432                         offset = -request;
433                         request_frames += request.frames (_film->audio_frame_rate ());
434                         if (request_frames < 0) {
435                                 request_frames = 0;
436                         }
437                         request = DCPTime ();
438                 }
439
440                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
441
442                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
443                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, request_frames, accurate);
444
445                 /* Gain */
446                 if (content->audio_gain() != 0) {
447                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
448                         gain->apply_gain (content->audio_gain ());
449                         all->audio = gain;
450                 }
451
452                 /* Remap channels */
453                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
454                 dcp_mapped->make_silent ();
455                 AudioMapping map = content->audio_mapping ();
456                 for (int i = 0; i < map.content_channels(); ++i) {
457                         for (int j = 0; j < _film->audio_channels(); ++j) {
458                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
459                                         dcp_mapped->accumulate_channel (
460                                                 all->audio.get(),
461                                                 i,
462                                                 j,
463                                                 map.get (i, static_cast<dcp::Channel> (j))
464                                                 );
465                                 }
466                         }
467                 }
468                 
469                 all->audio = dcp_mapped;
470
471                 audio->accumulate_frames (
472                         all->audio.get(),
473                         content_frame - all->frame,
474                         offset.frames (_film->audio_frame_rate()),
475                         min (AudioFrame (all->audio->frames()), request_frames)
476                         );
477         }
478
479         return audio;
480 }
481
482 VideoFrame
483 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
484 {
485         /* s is the offset of t from the start position of this content */
486         DCPTime s = t - piece->content->position ();
487         s = DCPTime (max (DCPTime::Type (0), s.get ()));
488         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
489
490         /* Convert this to the content frame */
491         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) / piece->frc.factor ();
492 }
493
494 DCPTime
495 Player::content_video_to_dcp (shared_ptr<const Piece> piece, VideoFrame f) const
496 {
497         DCPTime t = DCPTime::from_frames (f * piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
498         if (t < DCPTime ()) {
499                 t = DCPTime ();
500         }
501
502         return t;
503 }
504
505 AudioFrame
506 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
507 {
508         /* s is the offset of t from the start position of this content */
509         DCPTime s = t - piece->content->position ();
510         s = DCPTime (max (DCPTime::Type (0), s.get ()));
511         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
512
513         /* Convert this to the content frame */
514         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
515 }
516
517 ContentTime
518 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
519 {
520         /* s is the offset of t from the start position of this content */
521         DCPTime s = t - piece->content->position ();
522         s = DCPTime (max (DCPTime::Type (0), s.get ()));
523         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
524
525         return ContentTime (s + piece->content->trim_start(), piece->frc);
526 }
527
528 void
529 PlayerStatistics::dump (shared_ptr<Log> log) const
530 {
531         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
532         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
533 }
534
535 PlayerStatistics const &
536 Player::statistics () const
537 {
538         return _statistics;
539 }
540
541 PlayerSubtitles
542 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
543 {
544         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
545
546         PlayerSubtitles ps (time, length);
547
548         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
549                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
550                 if (!subtitle_content->use_subtitles ()) {
551                         continue;
552                 }
553
554                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
555                 ContentTime const from = dcp_to_content_subtitle (*j, time);
556                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
557                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
558
559                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
560                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
561                         
562                         /* Apply content's subtitle offsets */
563                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
564                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
565
566                         /* Apply content's subtitle scale */
567                         i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
568                         i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
569
570                         /* Apply a corrective translation to keep the subtitle centred after that scale */
571                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
572                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
573                         
574                         ps.image.push_back (i->sub);
575                 }
576
577                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
578                 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
579                         BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
580                                 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
581                                 s.set_size (s.size() * max (subtitle_content->subtitle_x_scale(), subtitle_content->subtitle_y_scale()));
582                                 ps.text.push_back (s);
583                         }
584                 }
585         }
586
587         return ps;
588 }
589
590 list<shared_ptr<Font> >
591 Player::get_subtitle_fonts ()
592 {
593         if (!_have_valid_pieces) {
594                 setup_pieces ();
595         }
596
597         list<shared_ptr<Font> > fonts;
598         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
599                 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
600                 if (sc) {
601                         /* XXX: things may go wrong if there are duplicate font IDs
602                            with different font files.
603                         */
604                         list<shared_ptr<Font> > f = sc->fonts ();
605                         copy (f.begin(), f.end(), back_inserter (fonts));
606                 }
607         }
608
609         return fonts;
610 }
611
612 /** Set this player never to produce any video data */
613 void
614 Player::set_ignore_video ()
615 {
616         _ignore_video = true;
617 }