Merge branch '2.0' of ssh://main.carlh.net/home/carl/git/dcpomatic into 2.0
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include "player.h"
21 #include "film.h"
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
33 #include "playlist.h"
34 #include "job.h"
35 #include "image.h"
36 #include "raw_image_proxy.h"
37 #include "ratio.h"
38 #include "log.h"
39 #include "render_subtitles.h"
40 #include "config.h"
41 #include "content_video.h"
42 #include "player_video.h"
43 #include "frame_rate_change.h"
44 #include "dcp_content.h"
45 #include "dcp_decoder.h"
46 #include "dcp_subtitle_content.h"
47 #include "dcp_subtitle_decoder.h"
48 #include <boost/foreach.hpp>
49 #include <stdint.h>
50 #include <algorithm>
51
52 #include "i18n.h"
53
54 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
55
56 using std::list;
57 using std::cout;
58 using std::min;
59 using std::max;
60 using std::min;
61 using std::vector;
62 using std::pair;
63 using std::map;
64 using std::make_pair;
65 using boost::shared_ptr;
66 using boost::weak_ptr;
67 using boost::dynamic_pointer_cast;
68 using boost::optional;
69
70 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
71         : _film (f)
72         , _playlist (p)
73         , _have_valid_pieces (false)
74         , _approximate_size (false)
75         , _ignore_video (false)
76 {
77         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
78         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
79         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
80         set_video_container_size (_film->frame_size ());
81 }
82
83 void
84 Player::setup_pieces ()
85 {
86         list<shared_ptr<Piece> > old_pieces = _pieces;
87         _pieces.clear ();
88
89         ContentList content = _playlist->content ();
90
91         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
92
93                 if (!(*i)->paths_valid ()) {
94                         continue;
95                 }
96                 
97                 shared_ptr<Decoder> decoder;
98                 optional<FrameRateChange> frc;
99
100                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
101                 DCPTime best_overlap_t;
102                 shared_ptr<VideoContent> best_overlap;
103                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
104                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
105                         if (!vc) {
106                                 continue;
107                         }
108                         
109                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
110                         if (overlap > best_overlap_t) {
111                                 best_overlap = vc;
112                                 best_overlap_t = overlap;
113                         }
114                 }
115
116                 optional<FrameRateChange> best_overlap_frc;
117                 if (best_overlap) {
118                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
119                 } else {
120                         /* No video overlap; e.g. if the DCP is just audio */
121                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
122                 }
123
124                 /* FFmpeg */
125                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
126                 if (fc) {
127                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
128                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
129                 }
130
131                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
132                 if (dc) {
133                         decoder.reset (new DCPDecoder (dc));
134                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
135                 }
136
137                 /* ImageContent */
138                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
139                 if (ic) {
140                         /* See if we can re-use an old ImageDecoder */
141                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
142                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
143                                 if (imd && imd->content() == ic) {
144                                         decoder = imd;
145                                 }
146                         }
147
148                         if (!decoder) {
149                                 decoder.reset (new ImageDecoder (ic));
150                         }
151
152                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
153                 }
154
155                 /* SndfileContent */
156                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
157                 if (sc) {
158                         decoder.reset (new SndfileDecoder (sc));
159                         frc = best_overlap_frc;
160                 }
161
162                 /* SubRipContent */
163                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
164                 if (rc) {
165                         decoder.reset (new SubRipDecoder (rc));
166                         frc = best_overlap_frc;
167                 }
168
169                 /* DCPSubtitleContent */
170                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
171                 if (dsc) {
172                         decoder.reset (new DCPSubtitleDecoder (dsc));
173                         frc = best_overlap_frc;
174                 }
175
176                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
177                 if (vd && _ignore_video) {
178                         vd->set_ignore_video ();
179                 }
180
181                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
182         }
183
184         _have_valid_pieces = true;
185 }
186
187 void
188 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
189 {
190         shared_ptr<Content> c = w.lock ();
191         if (!c) {
192                 return;
193         }
194
195         if (
196                 property == ContentProperty::POSITION ||
197                 property == ContentProperty::LENGTH ||
198                 property == ContentProperty::TRIM_START ||
199                 property == ContentProperty::TRIM_END ||
200                 property == ContentProperty::PATH ||
201                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
202                 property == DCPContentProperty::CAN_BE_PLAYED
203                 ) {
204                 
205                 _have_valid_pieces = false;
206                 Changed (frequent);
207
208         } else if (
209                 property == SubtitleContentProperty::USE_SUBTITLES ||
210                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
211                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
212                 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
213                 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
214                 property == VideoContentProperty::VIDEO_CROP ||
215                 property == VideoContentProperty::VIDEO_SCALE ||
216                 property == VideoContentProperty::VIDEO_FRAME_RATE ||
217                 property == VideoContentProperty::VIDEO_FADE_IN ||
218                 property == VideoContentProperty::VIDEO_FADE_OUT
219                 ) {
220                 
221                 Changed (frequent);
222         }
223 }
224
225 void
226 Player::playlist_changed ()
227 {
228         _have_valid_pieces = false;
229         Changed (false);
230 }
231
232 void
233 Player::set_video_container_size (dcp::Size s)
234 {
235         _video_container_size = s;
236
237         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
238         _black_image->make_black ();
239 }
240
241 void
242 Player::film_changed (Film::Property p)
243 {
244         /* Here we should notice Film properties that affect our output, and
245            alert listeners that our output now would be different to how it was
246            last time we were run.
247         */
248
249         if (p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
250                 Changed (false);
251         }
252 }
253
254 list<PositionImage>
255 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
256 {
257         list<PositionImage> all;
258         
259         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
260                 if (!i->image) {
261                         continue;
262                 }
263
264                 /* We will scale the subtitle up to fit _video_container_size */
265                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
266                 
267                 /* Then we need a corrective translation, consisting of two parts:
268                  *
269                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
270                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
271                  *
272                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
273                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
274                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
275                  *
276                  * Combining these two translations gives these expressions.
277                  */
278
279                 all.push_back (
280                         PositionImage (
281                                 i->image->scale (
282                                         scaled_size,
283                                         dcp::YUV_TO_RGB_REC601,
284                                         i->image->pixel_format (),
285                                         true
286                                         ),
287                                 Position<int> (
288                                         rint (_video_container_size.width * i->rectangle.x),
289                                         rint (_video_container_size.height * i->rectangle.y)
290                                         )
291                                 )
292                         );
293         }
294
295         return all;
296 }
297
298 void
299 Player::set_approximate_size ()
300 {
301         _approximate_size = true;
302 }
303
304 shared_ptr<PlayerVideo>
305 Player::black_player_video_frame (DCPTime time) const
306 {
307         return shared_ptr<PlayerVideo> (
308                 new PlayerVideo (
309                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
310                         time,
311                         Crop (),
312                         optional<float> (),
313                         _video_container_size,
314                         _video_container_size,
315                         EYES_BOTH,
316                         PART_WHOLE,
317                         Config::instance()->colour_conversions().front().conversion
318                 )
319         );
320 }
321
322 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
323 list<shared_ptr<PlayerVideo> >
324 Player::get_video (DCPTime time, bool accurate)
325 {
326         if (!_have_valid_pieces) {
327                 setup_pieces ();
328         }
329
330         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
331                 time,
332                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
333                 );
334
335         list<shared_ptr<PlayerVideo> > pvf;
336
337         if (ov.empty ()) {
338                 /* No video content at this time */
339                 pvf.push_back (black_player_video_frame (time));
340         } else {
341                 /* Create a PlayerVideo from the content's video at this time */
342
343                 shared_ptr<Piece> piece = ov.back ();
344                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
345                 DCPOMATIC_ASSERT (decoder);
346                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
347                 DCPOMATIC_ASSERT (content);
348
349                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
350                 if (content_video.empty ()) {
351                         pvf.push_back (black_player_video_frame (time));
352                         return pvf;
353                 }
354                 
355                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size (), _approximate_size ? 4 : 1);
356
357                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
358                         pvf.push_back (
359                                 shared_ptr<PlayerVideo> (
360                                         new PlayerVideo (
361                                                 i->image,
362                                                 content_video_to_dcp (piece, i->frame),
363                                                 content->crop (),
364                                                 content->fade (i->frame),
365                                                 image_size,
366                                                 _video_container_size,
367                                                 i->eyes,
368                                                 i->part,
369                                                 content->colour_conversion ()
370                                                 )
371                                         )
372                                 );
373                 }
374         }
375
376         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
377
378         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
379
380         list<PositionImage> sub_images;
381
382         /* Image subtitles */
383         list<PositionImage> c = transform_image_subtitles (ps.image);
384         copy (c.begin(), c.end(), back_inserter (sub_images));
385
386         /* Text subtitles (rendered to images) */
387         sub_images.push_back (render_subtitles (ps.text, _video_container_size));
388         
389         if (!sub_images.empty ()) {
390                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
391                         (*i)->set_subtitle (merge (sub_images));
392                 }
393         }       
394                 
395         return pvf;
396 }
397
398 shared_ptr<AudioBuffers>
399 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
400 {
401         if (!_have_valid_pieces) {
402                 setup_pieces ();
403         }
404
405         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
406
407         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
408         audio->make_silent ();
409         
410         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
411         if (ov.empty ()) {
412                 return audio;
413         }
414
415         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
416
417                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
418                 DCPOMATIC_ASSERT (content);
419                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
420                 DCPOMATIC_ASSERT (decoder);
421
422                 if (content->audio_frame_rate() == 0) {
423                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
424                          * audio stream).
425                          */
426                         continue;
427                 }
428
429                 /* The time that we should request from the content */
430                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
431                 AudioFrame request_frames = length_frames;
432                 DCPTime offset;
433                 if (request < DCPTime ()) {
434                         /* We went off the start of the content, so we will need to offset
435                            the stuff we get back.
436                         */
437                         offset = -request;
438                         request_frames += request.frames (_film->audio_frame_rate ());
439                         if (request_frames < 0) {
440                                 request_frames = 0;
441                         }
442                         request = DCPTime ();
443                 }
444
445                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
446
447                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
448                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, request_frames, accurate);
449
450                 /* Gain */
451                 if (content->audio_gain() != 0) {
452                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
453                         gain->apply_gain (content->audio_gain ());
454                         all->audio = gain;
455                 }
456
457                 /* Remap channels */
458                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
459                 dcp_mapped->make_silent ();
460                 AudioMapping map = content->audio_mapping ();
461                 for (int i = 0; i < map.content_channels(); ++i) {
462                         for (int j = 0; j < _film->audio_channels(); ++j) {
463                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
464                                         dcp_mapped->accumulate_channel (
465                                                 all->audio.get(),
466                                                 i,
467                                                 j,
468                                                 map.get (i, static_cast<dcp::Channel> (j))
469                                                 );
470                                 }
471                         }
472                 }
473                 
474                 all->audio = dcp_mapped;
475
476                 audio->accumulate_frames (
477                         all->audio.get(),
478                         content_frame - all->frame,
479                         offset.frames (_film->audio_frame_rate()),
480                         min (AudioFrame (all->audio->frames()), request_frames)
481                         );
482         }
483
484         return audio;
485 }
486
487 VideoFrame
488 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
489 {
490         /* s is the offset of t from the start position of this content */
491         DCPTime s = t - piece->content->position ();
492         s = DCPTime (max (DCPTime::Type (0), s.get ()));
493         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
494
495         /* Convert this to the content frame */
496         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) / piece->frc.factor ();
497 }
498
499 DCPTime
500 Player::content_video_to_dcp (shared_ptr<const Piece> piece, VideoFrame f) const
501 {
502         DCPTime t = DCPTime::from_frames (f * piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
503         if (t < DCPTime ()) {
504                 t = DCPTime ();
505         }
506
507         return t;
508 }
509
510 AudioFrame
511 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
512 {
513         /* s is the offset of t from the start position of this content */
514         DCPTime s = t - piece->content->position ();
515         s = DCPTime (max (DCPTime::Type (0), s.get ()));
516         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
517
518         /* Convert this to the content frame */
519         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
520 }
521
522 ContentTime
523 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
524 {
525         /* s is the offset of t from the start position of this content */
526         DCPTime s = t - piece->content->position ();
527         s = DCPTime (max (DCPTime::Type (0), s.get ()));
528         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
529
530         return ContentTime (s + piece->content->trim_start(), piece->frc);
531 }
532
533 void
534 PlayerStatistics::dump (shared_ptr<Log> log) const
535 {
536         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
537         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
538 }
539
540 PlayerStatistics const &
541 Player::statistics () const
542 {
543         return _statistics;
544 }
545
546 PlayerSubtitles
547 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
548 {
549         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
550
551         PlayerSubtitles ps (time, length);
552
553         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
554                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
555                 if (!subtitle_content->use_subtitles ()) {
556                         continue;
557                 }
558
559                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
560                 ContentTime const from = dcp_to_content_subtitle (*j, time);
561                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
562                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
563
564                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
565                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
566                         
567                         /* Apply content's subtitle offsets */
568                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
569                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
570
571                         /* Apply content's subtitle scale */
572                         i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
573                         i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
574
575                         /* Apply a corrective translation to keep the subtitle centred after that scale */
576                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
577                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
578                         
579                         ps.image.push_back (i->sub);
580                 }
581
582                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
583                 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
584                         BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
585                                 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
586                                 s.set_size (s.size() * max (subtitle_content->subtitle_x_scale(), subtitle_content->subtitle_y_scale()));
587                                 ps.text.push_back (s);
588                         }
589                 }
590         }
591
592         return ps;
593 }
594
595 list<shared_ptr<Font> >
596 Player::get_subtitle_fonts ()
597 {
598         if (!_have_valid_pieces) {
599                 setup_pieces ();
600         }
601
602         list<shared_ptr<Font> > fonts;
603         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
604                 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
605                 if (sc) {
606                         /* XXX: things may go wrong if there are duplicate font IDs
607                            with different font files.
608                         */
609                         list<shared_ptr<Font> > f = sc->fonts ();
610                         copy (f.begin(), f.end(), back_inserter (fonts));
611                 }
612         }
613
614         return fonts;
615 }
616
617 /** Set this player never to produce any video data */
618 void
619 Player::set_ignore_video ()
620 {
621         _ignore_video = true;
622 }