Apply single-processor branch manually; processor is now in Film, not AudioContent.
[dcpomatic.git] / src / lib / player.cc
1 /*
2     Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
3
4     This program is free software; you can redistribute it and/or modify
5     it under the terms of the GNU General Public License as published by
6     the Free Software Foundation; either version 2 of the License, or
7     (at your option) any later version.
8
9     This program is distributed in the hope that it will be useful,
10     but WITHOUT ANY WARRANTY; without even the implied warranty of
11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12     GNU General Public License for more details.
13
14     You should have received a copy of the GNU General Public License
15     along with this program; if not, write to the Free Software
16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 */
19
20 #include "player.h"
21 #include "film.h"
22 #include "ffmpeg_decoder.h"
23 #include "audio_buffers.h"
24 #include "ffmpeg_content.h"
25 #include "image_decoder.h"
26 #include "image_content.h"
27 #include "sndfile_decoder.h"
28 #include "sndfile_content.h"
29 #include "subtitle_content.h"
30 #include "subrip_decoder.h"
31 #include "subrip_content.h"
32 #include "dcp_content.h"
33 #include "playlist.h"
34 #include "job.h"
35 #include "image.h"
36 #include "raw_image_proxy.h"
37 #include "ratio.h"
38 #include "log.h"
39 #include "render_subtitles.h"
40 #include "config.h"
41 #include "content_video.h"
42 #include "player_video.h"
43 #include "frame_rate_change.h"
44 #include "dcp_content.h"
45 #include "dcp_decoder.h"
46 #include "dcp_subtitle_content.h"
47 #include "dcp_subtitle_decoder.h"
48 #include "audio_processor.h"
49 #include <boost/foreach.hpp>
50 #include <stdint.h>
51 #include <algorithm>
52
53 #include "i18n.h"
54
55 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
56
57 using std::list;
58 using std::cout;
59 using std::min;
60 using std::max;
61 using std::min;
62 using std::vector;
63 using std::pair;
64 using std::map;
65 using std::make_pair;
66 using boost::shared_ptr;
67 using boost::weak_ptr;
68 using boost::dynamic_pointer_cast;
69 using boost::optional;
70
71 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
72         : _film (f)
73         , _playlist (p)
74         , _have_valid_pieces (false)
75         , _ignore_video (false)
76 {
77         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
78         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
79         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
80         set_video_container_size (_film->frame_size ());
81
82         film_changed (Film::AUDIO_PROCESSOR);
83 }
84
85 void
86 Player::setup_pieces ()
87 {
88         list<shared_ptr<Piece> > old_pieces = _pieces;
89         _pieces.clear ();
90
91         ContentList content = _playlist->content ();
92
93         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
94
95                 if (!(*i)->paths_valid ()) {
96                         continue;
97                 }
98                 
99                 shared_ptr<Decoder> decoder;
100                 optional<FrameRateChange> frc;
101
102                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
103                 DCPTime best_overlap_t;
104                 shared_ptr<VideoContent> best_overlap;
105                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
106                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
107                         if (!vc) {
108                                 continue;
109                         }
110                         
111                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
112                         if (overlap > best_overlap_t) {
113                                 best_overlap = vc;
114                                 best_overlap_t = overlap;
115                         }
116                 }
117
118                 optional<FrameRateChange> best_overlap_frc;
119                 if (best_overlap) {
120                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
121                 } else {
122                         /* No video overlap; e.g. if the DCP is just audio */
123                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
124                 }
125
126                 /* FFmpeg */
127                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
128                 if (fc) {
129                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
130                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
131                 }
132
133                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
134                 if (dc) {
135                         decoder.reset (new DCPDecoder (dc));
136                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
137                 }
138
139                 /* ImageContent */
140                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
141                 if (ic) {
142                         /* See if we can re-use an old ImageDecoder */
143                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
144                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
145                                 if (imd && imd->content() == ic) {
146                                         decoder = imd;
147                                 }
148                         }
149
150                         if (!decoder) {
151                                 decoder.reset (new ImageDecoder (ic));
152                         }
153
154                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
155                 }
156
157                 /* SndfileContent */
158                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
159                 if (sc) {
160                         decoder.reset (new SndfileDecoder (sc));
161                         frc = best_overlap_frc;
162                 }
163
164                 /* SubRipContent */
165                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
166                 if (rc) {
167                         decoder.reset (new SubRipDecoder (rc));
168                         frc = best_overlap_frc;
169                 }
170
171                 /* DCPSubtitleContent */
172                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
173                 if (dsc) {
174                         decoder.reset (new DCPSubtitleDecoder (dsc));
175                         frc = best_overlap_frc;
176                 }
177
178                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> (decoder);
179                 if (vd && _ignore_video) {
180                         vd->set_ignore_video ();
181                 }
182
183                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
184         }
185
186         _have_valid_pieces = true;
187 }
188
189 void
190 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
191 {
192         shared_ptr<Content> c = w.lock ();
193         if (!c) {
194                 return;
195         }
196
197         if (
198                 property == ContentProperty::POSITION ||
199                 property == ContentProperty::LENGTH ||
200                 property == ContentProperty::TRIM_START ||
201                 property == ContentProperty::TRIM_END ||
202                 property == ContentProperty::PATH ||
203                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
204                 property == DCPContentProperty::CAN_BE_PLAYED
205                 ) {
206                 
207                 _have_valid_pieces = false;
208                 Changed (frequent);
209
210         } else if (
211                 property == SubtitleContentProperty::USE_SUBTITLES ||
212                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
213                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
214                 property == SubtitleContentProperty::SUBTITLE_X_SCALE ||
215                 property == SubtitleContentProperty::SUBTITLE_Y_SCALE ||
216                 property == VideoContentProperty::VIDEO_CROP ||
217                 property == VideoContentProperty::VIDEO_SCALE ||
218                 property == VideoContentProperty::VIDEO_FRAME_RATE ||
219                 property == VideoContentProperty::VIDEO_FADE_IN ||
220                 property == VideoContentProperty::VIDEO_FADE_OUT
221                 ) {
222                 
223                 Changed (frequent);
224         }
225 }
226
227 void
228 Player::playlist_changed ()
229 {
230         _have_valid_pieces = false;
231         Changed (false);
232 }
233
234 void
235 Player::set_video_container_size (dcp::Size s)
236 {
237         _video_container_size = s;
238
239         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
240         _black_image->make_black ();
241 }
242
243 void
244 Player::film_changed (Film::Property p)
245 {
246         /* Here we should notice Film properties that affect our output, and
247            alert listeners that our output now would be different to how it was
248            last time we were run.
249         */
250
251         if (p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
252                 Changed (false);
253         } else if (p == Film::AUDIO_PROCESSOR) {
254                 if (_film->audio_processor ()) {
255                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
256                 }
257         }
258 }
259
260 list<PositionImage>
261 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
262 {
263         list<PositionImage> all;
264         
265         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
266                 if (!i->image) {
267                         continue;
268                 }
269
270                 /* We will scale the subtitle up to fit _video_container_size */
271                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
272                 
273                 /* Then we need a corrective translation, consisting of two parts:
274                  *
275                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
276                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
277                  *
278                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
279                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
280                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
281                  *
282                  * Combining these two translations gives these expressions.
283                  */
284
285                 all.push_back (
286                         PositionImage (
287                                 i->image->scale (
288                                         scaled_size,
289                                         dcp::YUV_TO_RGB_REC601,
290                                         i->image->pixel_format (),
291                                         true
292                                         ),
293                                 Position<int> (
294                                         rint (_video_container_size.width * i->rectangle.x),
295                                         rint (_video_container_size.height * i->rectangle.y)
296                                         )
297                                 )
298                         );
299         }
300
301         return all;
302 }
303
304 shared_ptr<PlayerVideo>
305 Player::black_player_video_frame (DCPTime time) const
306 {
307         return shared_ptr<PlayerVideo> (
308                 new PlayerVideo (
309                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
310                         time,
311                         Crop (),
312                         optional<float> (),
313                         _video_container_size,
314                         _video_container_size,
315                         EYES_BOTH,
316                         PART_WHOLE,
317                         PresetColourConversion::all().front().conversion
318                 )
319         );
320 }
321
322 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
323 list<shared_ptr<PlayerVideo> >
324 Player::get_video (DCPTime time, bool accurate)
325 {
326         if (!_have_valid_pieces) {
327                 setup_pieces ();
328         }
329
330         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
331                 time,
332                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
333                 );
334
335         list<shared_ptr<PlayerVideo> > pvf;
336
337         if (ov.empty ()) {
338                 /* No video content at this time */
339                 pvf.push_back (black_player_video_frame (time));
340         } else {
341                 /* Create a PlayerVideo from the content's video at this time */
342
343                 shared_ptr<Piece> piece = ov.back ();
344                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
345                 DCPOMATIC_ASSERT (decoder);
346                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
347                 DCPOMATIC_ASSERT (content);
348
349                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
350                 if (content_video.empty ()) {
351                         pvf.push_back (black_player_video_frame (time));
352                         return pvf;
353                 }
354                 
355                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
356
357                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
358                         pvf.push_back (
359                                 shared_ptr<PlayerVideo> (
360                                         new PlayerVideo (
361                                                 i->image,
362                                                 content_video_to_dcp (piece, i->frame),
363                                                 content->crop (),
364                                                 content->fade (i->frame),
365                                                 image_size,
366                                                 _video_container_size,
367                                                 i->eyes,
368                                                 i->part,
369                                                 content->colour_conversion ()
370                                                 )
371                                         )
372                                 );
373                 }
374         }
375
376         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
377
378         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
379
380         list<PositionImage> sub_images;
381
382         /* Image subtitles */
383         list<PositionImage> c = transform_image_subtitles (ps.image);
384         copy (c.begin(), c.end(), back_inserter (sub_images));
385
386         /* Text subtitles (rendered to an image) */
387         if (!ps.text.empty ()) {
388                 sub_images.push_back (render_subtitles (ps.text, _video_container_size));
389         }
390
391         if (!sub_images.empty ()) {
392                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
393                         (*i)->set_subtitle (merge (sub_images));
394                 }
395         }       
396                 
397         return pvf;
398 }
399
400 shared_ptr<AudioBuffers>
401 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
402 {
403         if (!_have_valid_pieces) {
404                 setup_pieces ();
405         }
406
407         Frame const length_frames = length.frames (_film->audio_frame_rate ());
408
409         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
410         audio->make_silent ();
411         
412         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
413         if (ov.empty ()) {
414                 return audio;
415         }
416
417         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
418
419                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
420                 DCPOMATIC_ASSERT (content);
421                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
422                 DCPOMATIC_ASSERT (decoder);
423
424                 /* The time that we should request from the content */
425                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
426                 Frame request_frames = length_frames;
427                 DCPTime offset;
428                 if (request < DCPTime ()) {
429                         /* We went off the start of the content, so we will need to offset
430                            the stuff we get back.
431                         */
432                         offset = -request;
433                         request_frames += request.frames (_film->audio_frame_rate ());
434                         if (request_frames < 0) {
435                                 request_frames = 0;
436                         }
437                         request = DCPTime ();
438                 }
439
440                 Frame const content_frame = dcp_to_content_audio (*i, request);
441
442                 BOOST_FOREACH (AudioStreamPtr j, content->audio_streams ()) {
443                         
444                         /* Audio from this piece's decoder stream (which might be more or less than what we asked for) */
445                         ContentAudio all = decoder->get_audio (j, content_frame, request_frames, accurate);
446
447                         /* Gain */
448                         if (content->audio_gain() != 0) {
449                                 shared_ptr<AudioBuffers> gain (new AudioBuffers (all.audio));
450                                 gain->apply_gain (content->audio_gain ());
451                                 all.audio = gain;
452                         }
453
454                         /* Remap channels */
455                         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all.audio->frames()));
456                         dcp_mapped->make_silent ();
457                         AudioMapping map = j->mapping ();
458                         for (int i = 0; i < map.content_channels(); ++i) {
459                                 for (int j = 0; j < _film->audio_channels(); ++j) {
460                                         if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
461                                                 dcp_mapped->accumulate_channel (
462                                                         all.audio.get(),
463                                                         i,
464                                                         j,
465                                                         map.get (i, static_cast<dcp::Channel> (j))
466                                                         );
467                                         }
468                                 }
469                         }
470
471                         if (_audio_processor) {
472                                 dcp_mapped = _audio_processor->run (dcp_mapped);
473                         }
474                 
475                         all.audio = dcp_mapped;
476
477                         audio->accumulate_frames (
478                                 all.audio.get(),
479                                 content_frame - all.frame,
480                                 offset.frames (_film->audio_frame_rate()),
481                                 min (Frame (all.audio->frames()), request_frames)
482                                 );
483                 }
484         }
485
486         return audio;
487 }
488
489 Frame
490 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
491 {
492         /* s is the offset of t from the start position of this content */
493         DCPTime s = t - piece->content->position ();
494         s = DCPTime (max (DCPTime::Type (0), s.get ()));
495         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
496
497         /* Convert this to the content frame */
498         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) / piece->frc.factor ();
499 }
500
501 DCPTime
502 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
503 {
504         DCPTime t = DCPTime::from_frames (f * piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
505         if (t < DCPTime ()) {
506                 t = DCPTime ();
507         }
508
509         return t;
510 }
511
512 Frame
513 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
514 {
515         /* s is the offset of t from the start position of this content */
516         DCPTime s = t - piece->content->position ();
517         s = DCPTime (max (DCPTime::Type (0), s.get ()));
518         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
519
520         /* Convert this to the content frame */
521         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
522 }
523
524 ContentTime
525 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
526 {
527         /* s is the offset of t from the start position of this content */
528         DCPTime s = t - piece->content->position ();
529         s = DCPTime (max (DCPTime::Type (0), s.get ()));
530         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
531
532         return ContentTime (s + piece->content->trim_start(), piece->frc);
533 }
534
535 void
536 PlayerStatistics::dump (shared_ptr<Log> log) const
537 {
538         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
539         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
540 }
541
542 PlayerStatistics const &
543 Player::statistics () const
544 {
545         return _statistics;
546 }
547
548 PlayerSubtitles
549 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
550 {
551         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
552
553         PlayerSubtitles ps (time, length);
554
555         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
556                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
557                 if (!subtitle_content->use_subtitles ()) {
558                         continue;
559                 }
560
561                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
562                 ContentTime const from = dcp_to_content_subtitle (*j, time);
563                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
564                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
565
566                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
567                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
568                         
569                         /* Apply content's subtitle offsets */
570                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
571                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
572
573                         /* Apply content's subtitle scale */
574                         i->sub.rectangle.width *= subtitle_content->subtitle_x_scale ();
575                         i->sub.rectangle.height *= subtitle_content->subtitle_y_scale ();
576
577                         /* Apply a corrective translation to keep the subtitle centred after that scale */
578                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_x_scale() - 1);
579                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_y_scale() - 1);
580                         
581                         ps.image.push_back (i->sub);
582                 }
583
584                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
585                 BOOST_FOREACH (ContentTextSubtitle& ts, text) {
586                         BOOST_FOREACH (dcp::SubtitleString& s, ts.subs) {
587                                 s.set_v_position (s.v_position() + subtitle_content->subtitle_y_offset ());
588                                 s.set_size (s.size() * max (subtitle_content->subtitle_x_scale(), subtitle_content->subtitle_y_scale()));
589                                 ps.text.push_back (s);
590                         }
591                 }
592         }
593
594         return ps;
595 }
596
597 list<shared_ptr<Font> >
598 Player::get_subtitle_fonts ()
599 {
600         if (!_have_valid_pieces) {
601                 setup_pieces ();
602         }
603
604         list<shared_ptr<Font> > fonts;
605         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
606                 shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (p->content);
607                 if (sc) {
608                         /* XXX: things may go wrong if there are duplicate font IDs
609                            with different font files.
610                         */
611                         list<shared_ptr<Font> > f = sc->fonts ();
612                         copy (f.begin(), f.end(), back_inserter (fonts));
613                 }
614         }
615
616         return fonts;
617 }
618
619 /** Set this player never to produce any video data */
620 void
621 Player::set_ignore_video ()
622 {
623         _ignore_video = true;
624 }