Remove erroneous comment.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include "player.h"
21 #include "film.h"
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
33 #include "playlist.h"
34 #include "job.h"
35 #include "image.h"
36 #include "raw_image_proxy.h"
37 #include "ratio.h"
38 #include "log.h"
39 #include "scaler.h"
40 #include "render_subtitles.h"
41 #include "config.h"
42 #include "content_video.h"
43 #include "player_video.h"
44 #include "frame_rate_change.h"
45 #include "dcp_content.h"
46 #include "dcp_decoder.h"
47 #include "dcp_subtitle_content.h"
48 #include "dcp_subtitle_decoder.h"
49 #include <boost/foreach.hpp>
50 #include <stdint.h>
51 #include <algorithm>
52
53 #include "i18n.h"
54
55 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
56
57 using std::list;
58 using std::cout;
59 using std::min;
60 using std::max;
61 using std::min;
62 using std::vector;
63 using std::pair;
64 using std::map;
65 using std::make_pair;
66 using boost::shared_ptr;
67 using boost::weak_ptr;
68 using boost::dynamic_pointer_cast;
69 using boost::optional;
70
71 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
72         : _film (f)
73         , _playlist (p)
74         , _have_valid_pieces (false)
75         , _approximate_size (false)
76         , _ignore_video (false)
77 {
78         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
79         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
80         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
81         set_video_container_size (_film->frame_size ());
82 }
83
84 void
85 Player::setup_pieces ()
86 {
87         list<shared_ptr<Piece> > old_pieces = _pieces;
88         _pieces.clear ();
89
90         ContentList content = _playlist->content ();
91
92         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
93
94                 if (!(*i)->paths_valid ()) {
95                         continue;
96                 }
97                 
98                 shared_ptr<Decoder> decoder;
99                 optional<FrameRateChange> frc;
100
101                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
102                 DCPTime best_overlap_t;
103                 shared_ptr<VideoContent> best_overlap;
104                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
105                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
106                         if (!vc) {
107                                 continue;
108                         }
109                         
110                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
111                         if (overlap > best_overlap_t) {
112                                 best_overlap = vc;
113                                 best_overlap_t = overlap;
114                         }
115                 }
116
117                 optional<FrameRateChange> best_overlap_frc;
118                 if (best_overlap) {
119                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
120                 } else {
121                         /* No video overlap; e.g. if the DCP is just audio */
122                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
123                 }
124
125                 /* FFmpeg */
126                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
127                 if (fc) {
128                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
129                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
130                 }
131
132                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
133                 if (dc) {
134                         decoder.reset (new DCPDecoder (dc));
135                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
136                 }
137
138                 /* ImageContent */
139                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
140                 if (ic) {
141                         /* See if we can re-use an old ImageDecoder */
142                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
143                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
144                                 if (imd && imd->content() == ic) {
145                                         decoder = imd;
146                                 }
147                         }
148
149                         if (!decoder) {
150                                 decoder.reset (new ImageDecoder (ic));
151                         }
152
153                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
154                 }
155
156                 /* SndfileContent */
157                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
158                 if (sc) {
159                         decoder.reset (new SndfileDecoder (sc));
160                         frc = best_overlap_frc;
161                 }
162
163                 /* SubRipContent */
164                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
165                 if (rc) {
166                         decoder.reset (new SubRipDecoder (rc));
167                         frc = best_overlap_frc;
168                 }
169
170                 /* DCPSubtitleContent */
171                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
172                 if (dsc) {
173                         decoder.reset (new DCPSubtitleDecoder (dsc));
174                         frc = best_overlap_frc;
175                 }
176
177                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
178                 if (vd && _ignore_video) {
179                         vd->set_ignore_video ();
180                 }
181
182                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
183         }
184
185         _have_valid_pieces = true;
186 }
187
188 void
189 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
190 {
191         shared_ptr<Content> c = w.lock ();
192         if (!c) {
193                 return;
194         }
195
196         if (
197                 property == ContentProperty::POSITION ||
198                 property == ContentProperty::LENGTH ||
199                 property == ContentProperty::TRIM_START ||
200                 property == ContentProperty::TRIM_END ||
201                 property == ContentProperty::PATH ||
202                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
203                 property == DCPContentProperty::CAN_BE_PLAYED
204                 ) {
205                 
206                 _have_valid_pieces = false;
207                 Changed (frequent);
208
209         } else if (
210                 property == SubtitleContentProperty::USE_SUBTITLES ||
211                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
212                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
213                 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
214                 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
215                 property == VideoContentProperty::VIDEO_CROP ||
216                 property == VideoContentProperty::VIDEO_SCALE ||
217                 property == VideoContentProperty::VIDEO_FRAME_RATE ||
218                 property == VideoContentProperty::VIDEO_FADE_IN ||
219                 property == VideoContentProperty::VIDEO_FADE_OUT
220                 ) {
221                 
222                 Changed (frequent);
223         }
224 }
225
226 void
227 Player::playlist_changed ()
228 {
229         _have_valid_pieces = false;
230         Changed (false);
231 }
232
233 void
234 Player::set_video_container_size (dcp::Size s)
235 {
236         _video_container_size = s;
237
238         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
239         _black_image->make_black ();
240 }
241
242 void
243 Player::film_changed (Film::Property p)
244 {
245         /* Here we should notice Film properties that affect our output, and
246            alert listeners that our output now would be different to how it was
247            last time we were run.
248         */
249
250         if (p == Film::SCALER || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
251                 Changed (false);
252         }
253 }
254
255 list<PositionImage>
256 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
257 {
258         list<PositionImage> all;
259         
260         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
261                 if (!i->image) {
262                         continue;
263                 }
264
265                 /* We will scale the subtitle up to fit _video_container_size */
266                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
267                 
268                 /* Then we need a corrective translation, consisting of two parts:
269                  *
270                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
271                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
272                  *
273                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
274                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
275                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
276                  *
277                  * Combining these two translations gives these expressions.
278                  */
279
280                 all.push_back (
281                         PositionImage (
282                                 i->image->scale (
283                                         scaled_size,
284                                         Scaler::from_id ("bicubic"),
285                                         i->image->pixel_format (),
286                                         true
287                                         ),
288                                 Position<int> (
289                                         rint (_video_container_size.width * i->rectangle.x),
290                                         rint (_video_container_size.height * i->rectangle.y)
291                                         )
292                                 )
293                         );
294         }
295
296         return all;
297 }
298
299 void
300 Player::set_approximate_size ()
301 {
302         _approximate_size = true;
303 }
304
305 shared_ptr<PlayerVideo>
306 Player::black_player_video_frame (DCPTime time) const
307 {
308         return shared_ptr<PlayerVideo> (
309                 new PlayerVideo (
310                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
311                         time,
312                         Crop (),
313                         optional<float> (),
314                         _video_container_size,
315                         _video_container_size,
316                         Scaler::from_id ("bicubic"),
317                         EYES_BOTH,
318                         PART_WHOLE,
319                         Config::instance()->colour_conversions().front().conversion
320                 )
321         );
322 }
323
324 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
325 list<shared_ptr<PlayerVideo> >
326 Player::get_video (DCPTime time, bool accurate)
327 {
328         if (!_have_valid_pieces) {
329                 setup_pieces ();
330         }
331
332         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
333                 time,
334                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
335                 );
336
337         list<shared_ptr<PlayerVideo> > pvf;
338
339         if (ov.empty ()) {
340                 /* No video content at this time */
341                 pvf.push_back (black_player_video_frame (time));
342         } else {
343                 /* Create a PlayerVideo from the content's video at this time */
344
345                 shared_ptr<Piece> piece = ov.back ();
346                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
347                 DCPOMATIC_ASSERT (decoder);
348                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
349                 DCPOMATIC_ASSERT (content);
350
351                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
352                 if (content_video.empty ()) {
353                         pvf.push_back (black_player_video_frame (time));
354                         return pvf;
355                 }
356                 
357                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size (), _approximate_size ? 4 : 1);
358
359                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
360                         pvf.push_back (
361                                 shared_ptr<PlayerVideo> (
362                                         new PlayerVideo (
363                                                 i->image,
364                                                 content_video_to_dcp (piece, i->frame),
365                                                 content->crop (),
366                                                 content->fade (i->frame),
367                                                 image_size,
368                                                 _video_container_size,
369                                                 _film->scaler(),
370                                                 i->eyes,
371                                                 i->part,
372                                                 content->colour_conversion ()
373                                                 )
374                                         )
375                                 );
376                 }
377         }
378
379         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
380
381         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
382
383         list<PositionImage> sub_images;
384
385         /* Image subtitles */
386         list<PositionImage> c = transform_image_subtitles (ps.image);
387         copy (c.begin(), c.end(), back_inserter (sub_images));
388
389         /* Text subtitles (rendered to images) */
390         sub_images.push_back (render_subtitles (ps.text, _video_container_size));
391         
392         if (!sub_images.empty ()) {
393                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
394                         (*i)->set_subtitle (merge (sub_images));
395                 }
396         }       
397                 
398         return pvf;
399 }
400
401 shared_ptr<AudioBuffers>
402 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
403 {
404         if (!_have_valid_pieces) {
405                 setup_pieces ();
406         }
407
408         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
409
410         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
411         audio->make_silent ();
412         
413         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
414         if (ov.empty ()) {
415                 return audio;
416         }
417
418         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
419
420                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
421                 DCPOMATIC_ASSERT (content);
422                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
423                 DCPOMATIC_ASSERT (decoder);
424
425                 if (content->audio_frame_rate() == 0) {
426                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
427                          * audio stream).
428                          */
429                         continue;
430                 }
431
432                 /* The time that we should request from the content */
433                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
434                 AudioFrame request_frames = length_frames;
435                 DCPTime offset;
436                 if (request < DCPTime ()) {
437                         /* We went off the start of the content, so we will need to offset
438                            the stuff we get back.
439                         */
440                         offset = -request;
441                         request_frames += request.frames (_film->audio_frame_rate ());
442                         if (request_frames < 0) {
443                                 request_frames = 0;
444                         }
445                         request = DCPTime ();
446                 }
447
448                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
449
450                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
451                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, request_frames, accurate);
452
453                 /* Gain */
454                 if (content->audio_gain() != 0) {
455                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
456                         gain->apply_gain (content->audio_gain ());
457                         all->audio = gain;
458                 }
459
460                 /* Remap channels */
461                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
462                 dcp_mapped->make_silent ();
463                 AudioMapping map = content->audio_mapping ();
464                 for (int i = 0; i < map.content_channels(); ++i) {
465                         for (int j = 0; j < _film->audio_channels(); ++j) {
466                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
467                                         dcp_mapped->accumulate_channel (
468                                                 all->audio.get(),
469                                                 i,
470                                                 j,
471                                                 map.get (i, static_cast<dcp::Channel> (j))
472                                                 );
473                                 }
474                         }
475                 }
476                 
477                 all->audio = dcp_mapped;
478
479                 audio->accumulate_frames (
480                         all->audio.get(),
481                         content_frame - all->frame,
482                         offset.frames (_film->audio_frame_rate()),
483                         min (AudioFrame (all->audio->frames()), request_frames)
484                         );
485         }
486
487         return audio;
488 }
489
490 VideoFrame
491 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
492 {
493         /* s is the offset of t from the start position of this content */
494         DCPTime s = t - piece->content->position ();
495         s = DCPTime (max (DCPTime::Type (0), s.get ()));
496         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
497
498         /* Convert this to the content frame */
499         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) / piece->frc.factor ();
500 }
501
502 DCPTime
503 Player::content_video_to_dcp (shared_ptr<const Piece> piece, VideoFrame f) const
504 {
505         DCPTime t = DCPTime::from_frames (f * piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
506         if (t < DCPTime ()) {
507                 t = DCPTime ();
508         }
509
510         return t;
511 }
512
513 AudioFrame
514 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
515 {
516         /* s is the offset of t from the start position of this content */
517         DCPTime s = t - piece->content->position ();
518         s = DCPTime (max (DCPTime::Type (0), s.get ()));
519         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
520
521         /* Convert this to the content frame */
522         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
523 }
524
525 ContentTime
526 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
527 {
528         /* s is the offset of t from the start position of this content */
529         DCPTime s = t - piece->content->position ();
530         s = DCPTime (max (DCPTime::Type (0), s.get ()));
531         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
532
533         return ContentTime (s + piece->content->trim_start(), piece->frc);
534 }
535
536 void
537 PlayerStatistics::dump (shared_ptr<Log> log) const
538 {
539         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
540         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
541 }
542
543 PlayerStatistics const &
544 Player::statistics () const
545 {
546         return _statistics;
547 }
548
549 PlayerSubtitles
550 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
551 {
552         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
553
554         PlayerSubtitles ps (time, length);
555
556         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
557                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
558                 if (!subtitle_content->use_subtitles ()) {
559                         continue;
560                 }
561
562                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
563                 ContentTime const from = dcp_to_content_subtitle (*j, time);
564                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
565                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
566
567                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
568                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
569                         
570                         /* Apply content's subtitle offsets */
571                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
572                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
573
574                         /* Apply content's subtitle scale */
575                         i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
576                         i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
577
578                         /* Apply a corrective translation to keep the subtitle centred after that scale */
579                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
580                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
581                         
582                         ps.image.push_back (i->sub);
583                 }
584
585                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
586                 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
587                         BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
588                                 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
589                                 s.set_size (s.size() * max (subtitle_content->subtitle_x_scale(), subtitle_content->subtitle_y_scale()));
590                                 ps.text.push_back (s);
591                         }
592                 }
593         }
594
595         return ps;
596 }
597
598 list<shared_ptr<Font> >
599 Player::get_subtitle_fonts ()
600 {
601         if (!_have_valid_pieces) {
602                 setup_pieces ();
603         }
604
605         list<shared_ptr<Font> > fonts;
606         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
607                 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
608                 if (sc) {
609                         /* XXX: things may go wrong if there are duplicate font IDs
610                            with different font files.
611                         */
612                         list<shared_ptr<Font> > f = sc->fonts ();
613                         copy (f.begin(), f.end(), back_inserter (fonts));
614                 }
615         }
616
617         return fonts;
618 }
619
620 /** Set this player never to produce any video data */
621 void
622 Player::set_ignore_video ()
623 {
624         _ignore_video = true;
625 }