src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2018 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_subtitles.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "subtitle_content.h"
  44 #include "subtitle_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "content_subtitle.h"
  48 #include "dcp_decoder.h"
  49 #include "image_decoder.h"
  50 #include "compose.hpp"
  51 #include "shuffler.h"
  52 #include <dcp/reel.h>
  53 #include <dcp/reel_sound_asset.h>
  54 #include <dcp/reel_subtitle_asset.h>
  55 #include <dcp/reel_picture_asset.h>
  56 #include <boost/foreach.hpp>
  57 #include <stdint.h>
  58 #include <algorithm>
  59 #include <iostream>
  60
  61 #include "i18n.h"
  62
  63 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
  64
  65 using std::list;
  66 using std::cout;
  67 using std::min;
  68 using std::max;
  69 using std::min;
  70 using std::vector;
  71 using std::pair;
  72 using std::map;
  73 using std::make_pair;
  74 using std::copy;
  75 using boost::shared_ptr;
  76 using boost::weak_ptr;
  77 using boost::dynamic_pointer_cast;
  78 using boost::optional;
  79 using boost::scoped_ptr;
  80
  81 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
  82 int const PlayerProperty::PLAYLIST = 701;
  83 int const PlayerProperty::FILM_CONTAINER = 702;
  84 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
  85 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
  86
  87 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  88         : _film (film)
  89         , _playlist (playlist)
  90         , _have_valid_pieces (false)
  91         , _ignore_video (false)
  92         , _ignore_subtitle (false)
  93         , _always_burn_subtitles (false)
  94         , _fast (false)
  95         , _play_referenced (false)
  96         , _audio_merger (_film->audio_frame_rate())
  97         , _shuffler (0)
  98 {
  99         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
 100         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 101         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
 102         set_video_container_size (_film->frame_size ());
 103
 104         film_changed (Film::AUDIO_PROCESSOR);
 105
 106         seek (DCPTime (), true);
 107 }
 108
 109 Player::~Player ()
 110 {
 111         delete _shuffler;
 112 }
 113
 114 void
 115 Player::setup_pieces ()
 116 {
 117         _pieces.clear ();
 118
 119         delete _shuffler;
 120         _shuffler = new Shuffler();
 121         _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
 122
 123         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 124
 125                 if (!i->paths_valid ()) {
 126                         continue;
 127                 }
 128
 129                 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log(), _fast);
 130                 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
 131
 132                 if (!decoder) {
 133                         /* Not something that we can decode; e.g. Atmos content */
 134                         continue;
 135                 }
 136
 137                 if (decoder->video && _ignore_video) {
 138                         decoder->video->set_ignore (true);
 139                 }
 140
 141                 if (decoder->subtitle && _ignore_subtitle) {
 142                         decoder->subtitle->set_ignore (true);
 143                 }
 144
 145                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 146                 if (dcp) {
 147                         dcp->set_decode_referenced (_play_referenced);
 148                         if (_play_referenced) {
 149                                 dcp->set_forced_reduction (_dcp_decode_reduction);
 150                         }
 151                 }
 152
 153                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 154                 _pieces.push_back (piece);
 155
 156                 if (decoder->video) {
 157                         if (i->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT || i->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) {
 158                                 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
 159                                 decoder->video->Data.connect (bind (&Shuffler::video, _shuffler, weak_ptr<Piece>(piece), _1));
 160                         } else {
 161                                 decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece>(piece), _1));
 162                         }
 163                 }
 164
 165                 if (decoder->audio) {
 166                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 167                 }
 168
 169                 if (decoder->subtitle) {
 170                         decoder->subtitle->ImageStart.connect (bind (&Player::image_subtitle_start, this, weak_ptr<Piece> (piece), _1));
 171                         decoder->subtitle->TextStart.connect (bind (&Player::text_subtitle_start, this, weak_ptr<Piece> (piece), _1));
 172                         decoder->subtitle->Stop.connect (bind (&Player::subtitle_stop, this, weak_ptr<Piece> (piece), _1));
 173                 }
 174         }
 175
 176         _stream_states.clear ();
 177         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 178                 if (i->content->audio) {
 179                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 180                                 _stream_states[j] = StreamState (i, i->content->position ());
 181                         }
 182                 }
 183         }
 184
 185         _black = Empty (_film->content(), _film->length(), bind(&Content::video, _1));
 186         _silent = Empty (_film->content(), _film->length(), bind(&Content::audio, _1));
 187
 188         _last_video_time = DCPTime ();
 189         _last_video_eyes = EYES_BOTH;
 190         _last_audio_time = DCPTime ();
 191         _have_valid_pieces = true;
 192 }
 193
 194 void
 195 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
 196 {
 197         shared_ptr<Content> c = w.lock ();
 198         if (!c) {
 199                 return;
 200         }
 201
 202         if (
 203                 property == ContentProperty::POSITION ||
 204                 property == ContentProperty::LENGTH ||
 205                 property == ContentProperty::TRIM_START ||
 206                 property == ContentProperty::TRIM_END ||
 207                 property == ContentProperty::PATH ||
 208                 property == VideoContentProperty::FRAME_TYPE ||
 209                 property == DCPContentProperty::NEEDS_ASSETS ||
 210                 property == DCPContentProperty::NEEDS_KDM ||
 211                 property == SubtitleContentProperty::COLOUR ||
 212                 property == SubtitleContentProperty::EFFECT ||
 213                 property == SubtitleContentProperty::EFFECT_COLOUR ||
 214                 property == FFmpegContentProperty::SUBTITLE_STREAM ||
 215                 property == FFmpegContentProperty::FILTERS ||
 216                 property == VideoContentProperty::COLOUR_CONVERSION
 217                 ) {
 218
 219                 _have_valid_pieces = false;
 220                 Changed (property, frequent);
 221
 222         } else if (
 223                 property == SubtitleContentProperty::LINE_SPACING ||
 224                 property == SubtitleContentProperty::OUTLINE_WIDTH ||
 225                 property == SubtitleContentProperty::Y_SCALE ||
 226                 property == SubtitleContentProperty::FADE_IN ||
 227                 property == SubtitleContentProperty::FADE_OUT ||
 228                 property == ContentProperty::VIDEO_FRAME_RATE ||
 229                 property == SubtitleContentProperty::USE ||
 230                 property == SubtitleContentProperty::X_OFFSET ||
 231                 property == SubtitleContentProperty::Y_OFFSET ||
 232                 property == SubtitleContentProperty::X_SCALE ||
 233                 property == SubtitleContentProperty::FONTS ||
 234                 property == VideoContentProperty::CROP ||
 235                 property == VideoContentProperty::SCALE ||
 236                 property == VideoContentProperty::FADE_IN ||
 237                 property == VideoContentProperty::FADE_OUT
 238                 ) {
 239
 240                 Changed (property, frequent);
 241         }
 242 }
 243
 244 void
 245 Player::set_video_container_size (dcp::Size s)
 246 {
 247         if (s == _video_container_size) {
 248                 return;
 249         }
 250
 251         _video_container_size = s;
 252
 253         _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 254         _black_image->make_black ();
 255
 256         Changed (PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 257 }
 258
 259 void
 260 Player::playlist_changed ()
 261 {
 262         _have_valid_pieces = false;
 263         Changed (PlayerProperty::PLAYLIST, false);
 264 }
 265
 266 void
 267 Player::film_changed (Film::Property p)
 268 {
 269         /* Here we should notice Film properties that affect our output, and
 270            alert listeners that our output now would be different to how it was
 271            last time we were run.
 272         */
 273
 274         if (p == Film::CONTAINER) {
 275                 Changed (PlayerProperty::FILM_CONTAINER, false);
 276         } else if (p == Film::VIDEO_FRAME_RATE) {
 277                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 278                    so we need new pieces here.
 279                 */
 280                 _have_valid_pieces = false;
 281                 Changed (PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
 282         } else if (p == Film::AUDIO_PROCESSOR) {
 283                 if (_film->audio_processor ()) {
 284                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 285                 }
 286         } else if (p == Film::AUDIO_CHANNELS) {
 287                 _audio_merger.clear ();
 288         }
 289 }
 290
 291 list<PositionImage>
 292 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
 293 {
 294         list<PositionImage> all;
 295
 296         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 297                 if (!i->image) {
 298                         continue;
 299                 }
 300
 301                 /* We will scale the subtitle up to fit _video_container_size */
 302                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
 303
 304                 all.push_back (
 305                         PositionImage (
 306                                 i->image->scale (
 307                                         scaled_size,
 308                                         dcp::YUV_TO_RGB_REC601,
 309                                         i->image->pixel_format (),
 310                                         true,
 311                                         _fast
 312                                         ),
 313                                 Position<int> (
 314                                         lrint (_video_container_size.width * i->rectangle.x),
 315                                         lrint (_video_container_size.height * i->rectangle.y)
 316                                         )
 317                                 )
 318                         );
 319         }
 320
 321         return all;
 322 }
 323
 324 shared_ptr<PlayerVideo>
 325 Player::black_player_video_frame (Eyes eyes) const
 326 {
 327         return shared_ptr<PlayerVideo> (
 328                 new PlayerVideo (
 329                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 330                         Crop (),
 331                         optional<double> (),
 332                         _video_container_size,
 333                         _video_container_size,
 334                         eyes,
 335                         PART_WHOLE,
 336                         PresetColourConversion::all().front().conversion,
 337                         boost::weak_ptr<Content>(),
 338                         boost::optional<Frame>()
 339                 )
 340         );
 341 }
 342
 343 Frame
 344 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 345 {
 346         DCPTime s = t - piece->content->position ();
 347         s = min (piece->content->length_after_trim(), s);
 348         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 349
 350         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 351            then convert that ContentTime to frames at the content's rate.  However this fails for
 352            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 353            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 354
 355            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 356         */
 357         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 358 }
 359
 360 DCPTime
 361 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 362 {
 363         /* See comment in dcp_to_content_video */
 364         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime(piece->content->trim_start(), piece->frc);
 365         return d + piece->content->position();
 366 }
 367
 368 Frame
 369 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 370 {
 371         DCPTime s = t - piece->content->position ();
 372         s = min (piece->content->length_after_trim(), s);
 373         /* See notes in dcp_to_content_video */
 374         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 375 }
 376
 377 DCPTime
 378 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 379 {
 380         /* See comment in dcp_to_content_video */
 381         return DCPTime::from_frames (f, _film->audio_frame_rate())
 382                 - DCPTime (piece->content->trim_start(), piece->frc)
 383                 + piece->content->position();
 384 }
 385
 386 ContentTime
 387 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 388 {
 389         DCPTime s = t - piece->content->position ();
 390         s = min (piece->content->length_after_trim(), s);
 391         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 392 }
 393
 394 DCPTime
 395 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 396 {
 397         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 398 }
 399
 400 list<shared_ptr<Font> >
 401 Player::get_subtitle_fonts ()
 402 {
 403         if (!_have_valid_pieces) {
 404                 setup_pieces ();
 405         }
 406
 407         list<shared_ptr<Font> > fonts;
 408         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
 409                 if (p->content->subtitle) {
 410                         /* XXX: things may go wrong if there are duplicate font IDs
 411                            with different font files.
 412                         */
 413                         list<shared_ptr<Font> > f = p->content->subtitle->fonts ();
 414                         copy (f.begin(), f.end(), back_inserter (fonts));
 415                 }
 416         }
 417
 418         return fonts;
 419 }
 420
 421 /** Set this player never to produce any video data */
 422 void
 423 Player::set_ignore_video ()
 424 {
 425         _ignore_video = true;
 426 }
 427
 428 void
 429 Player::set_ignore_subtitle ()
 430 {
 431         _ignore_subtitle = true;
 432 }
 433
 434 /** Set whether or not this player should always burn text subtitles into the image,
 435  *  regardless of the content settings.
 436  *  @param burn true to always burn subtitles, false to obey content settings.
 437  */
 438 void
 439 Player::set_always_burn_subtitles (bool burn)
 440 {
 441         _always_burn_subtitles = burn;
 442 }
 443
 444 /** Sets up the player to be faster, possibly at the expense of quality */
 445 void
 446 Player::set_fast ()
 447 {
 448         _fast = true;
 449         _have_valid_pieces = false;
 450 }
 451
 452 void
 453 Player::set_play_referenced ()
 454 {
 455         _play_referenced = true;
 456         _have_valid_pieces = false;
 457 }
 458
 459 list<ReferencedReelAsset>
 460 Player::get_reel_assets ()
 461 {
 462         list<ReferencedReelAsset> a;
 463
 464         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 465                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 466                 if (!j) {
 467                         continue;
 468                 }
 469
 470                 scoped_ptr<DCPDecoder> decoder;
 471                 try {
 472                         decoder.reset (new DCPDecoder (j, _film->log(), false));
 473                 } catch (...) {
 474                         return a;
 475                 }
 476
 477                 int64_t offset = 0;
 478                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 479
 480                         DCPOMATIC_ASSERT (j->video_frame_rate ());
 481                         double const cfr = j->video_frame_rate().get();
 482                         Frame const trim_start = j->trim_start().frames_round (cfr);
 483                         Frame const trim_end = j->trim_end().frames_round (cfr);
 484                         int const ffr = _film->video_frame_rate ();
 485
 486                         DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
 487                         if (j->reference_video ()) {
 488                                 shared_ptr<dcp::ReelAsset> ra = k->main_picture ();
 489                                 DCPOMATIC_ASSERT (ra);
 490                                 ra->set_entry_point (ra->entry_point() + trim_start);
 491                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 492                                 a.push_back (
 493                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 494                                         );
 495                         }
 496
 497                         if (j->reference_audio ()) {
 498                                 shared_ptr<dcp::ReelAsset> ra = k->main_sound ();
 499                                 DCPOMATIC_ASSERT (ra);
 500                                 ra->set_entry_point (ra->entry_point() + trim_start);
 501                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 502                                 a.push_back (
 503                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 504                                         );
 505                         }
 506
 507                         if (j->reference_subtitle ()) {
 508                                 shared_ptr<dcp::ReelAsset> ra = k->main_subtitle ();
 509                                 DCPOMATIC_ASSERT (ra);
 510                                 ra->set_entry_point (ra->entry_point() + trim_start);
 511                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 512                                 a.push_back (
 513                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 514                                         );
 515                         }
 516
 517                         /* Assume that main picture duration is the length of the reel */
 518                         offset += k->main_picture()->duration ();
 519                 }
 520         }
 521
 522         return a;
 523 }
 524
 525 bool
 526 Player::pass ()
 527 {
 528         if (!_have_valid_pieces) {
 529                 setup_pieces ();
 530         }
 531
 532         if (_playlist->length() == DCPTime()) {
 533                 /* Special case of an empty Film; just give one black frame */
 534                 emit_video (black_player_video_frame(EYES_BOTH), DCPTime());
 535                 return true;
 536         }
 537
 538         /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
 539
 540         shared_ptr<Piece> earliest_content;
 541         optional<DCPTime> earliest_time;
 542
 543         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 544                 if (i->done) {
 545                         continue;
 546                 }
 547
 548                 DCPTime const t = content_time_to_dcp (i, i->decoder->position());
 549                 if (t > i->content->end()) {
 550                         i->done = true;
 551                 } else {
 552
 553                         /* Given two choices at the same time, pick the one with a subtitle so we see it before
 554                            the video.
 555                         */
 556                         if (!earliest_time || t < *earliest_time || (t == *earliest_time && i->decoder->subtitle)) {
 557                                 earliest_time = t;
 558                                 earliest_content = i;
 559                         }
 560                 }
 561         }
 562
 563         bool done = false;
 564
 565         enum {
 566                 NONE,
 567                 CONTENT,
 568                 BLACK,
 569                 SILENT
 570         } which = NONE;
 571
 572         if (earliest_content) {
 573                 which = CONTENT;
 574         }
 575
 576         if (!_black.done() && (!earliest_time || _black.position() < *earliest_time)) {
 577                 earliest_time = _black.position ();
 578                 which = BLACK;
 579         }
 580
 581         if (!_silent.done() && (!earliest_time || _silent.position() < *earliest_time)) {
 582                 earliest_time = _silent.position ();
 583                 which = SILENT;
 584         }
 585
 586         switch (which) {
 587         case CONTENT:
 588                 earliest_content->done = earliest_content->decoder->pass ();
 589                 break;
 590         case BLACK:
 591                 emit_video (black_player_video_frame(EYES_BOTH), _black.position());
 592                 _black.set_position (_black.position() + one_video_frame());
 593                 break;
 594         case SILENT:
 595         {
 596                 DCPTimePeriod period (_silent.period_at_position());
 597                 if (_last_audio_time) {
 598                         /* Sometimes the thing that happened last finishes fractionally before
 599                            this silence.  Bodge the start time of the silence to fix it.  I'm
 600                            not sure if this is the right solution --- maybe the last thing should
 601                            be padded `forward' rather than this thing padding `back'.
 602                         */
 603                         period.from = min(period.from, *_last_audio_time);
 604                 }
 605                 if (period.duration() > one_video_frame()) {
 606                         period.to = period.from + one_video_frame();
 607                 }
 608                 fill_audio (period);
 609                 _silent.set_position (period.to);
 610                 break;
 611         }
 612         case NONE:
 613                 done = true;
 614                 break;
 615         }
 616
 617         /* Emit any audio that is ready */
 618
 619         /* Work out the time before which the audio is definitely all here.  This is the earliest last_push_end of one
 620            of our streams, or the position of the _silent.
 621         */
 622         DCPTime pull_to = _film->length ();
 623         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 624                 if (!i->second.piece->done && i->second.last_push_end < pull_to) {
 625                         pull_to = i->second.last_push_end;
 626                 }
 627         }
 628         if (!_silent.done() && _silent.position() < pull_to) {
 629                 pull_to = _silent.position();
 630         }
 631
 632         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 633         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 634                 if (_last_audio_time && i->second < *_last_audio_time) {
 635                         /* This new data comes before the last we emitted (or the last seek); discard it */
 636                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 637                         if (!cut.first) {
 638                                 continue;
 639                         }
 640                         *i = cut;
 641                 } else if (_last_audio_time && i->second > *_last_audio_time) {
 642                         /* There's a gap between this data and the last we emitted; fill with silence */
 643                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 644                 }
 645
 646                 emit_audio (i->first, i->second);
 647         }
 648
 649         if (done) {
 650                 _shuffler->flush ();
 651                 for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 652                         do_emit_video(i->first, i->second);
 653                 }
 654         }
 655
 656         return done;
 657 }
 658
 659 optional<PositionImage>
 660 Player::subtitles_for_frame (DCPTime time) const
 661 {
 662         list<PositionImage> subtitles;
 663
 664         int const vfr = _film->video_frame_rate();
 665
 666         BOOST_FOREACH (PlayerSubtitles i, _active_subtitles.get_burnt (DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), _always_burn_subtitles)) {
 667
 668                 /* Image subtitles */
 669                 list<PositionImage> c = transform_image_subtitles (i.image);
 670                 copy (c.begin(), c.end(), back_inserter (subtitles));
 671
 672                 /* Text subtitles (rendered to an image) */
 673                 if (!i.text.empty ()) {
 674                         list<PositionImage> s = render_subtitles (i.text, i.fonts, _video_container_size, time, vfr);
 675                         copy (s.begin(), s.end(), back_inserter (subtitles));
 676                 }
 677         }
 678
 679         if (subtitles.empty ()) {
 680                 return optional<PositionImage> ();
 681         }
 682
 683         return merge (subtitles);
 684 }
 685
 686 void
 687 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 688 {
 689         shared_ptr<Piece> piece = wp.lock ();
 690         if (!piece) {
 691                 return;
 692         }
 693
 694         FrameRateChange frc(piece->content->active_video_frame_rate(), _film->video_frame_rate());
 695         if (frc.skip && (video.frame % 2) == 1) {
 696                 return;
 697         }
 698
 699         /* Time of the first frame we will emit */
 700         DCPTime const time = content_video_to_dcp (piece, video.frame);
 701
 702         /* Discard if it's before the content's period or the last accurate seek.  We can't discard
 703            if it's after the content's period here as in that case we still need to fill any gap between
 704            `now' and the end of the content's period.
 705         */
 706         if (time < piece->content->position() || (_last_video_time && time < *_last_video_time)) {
 707                 return;
 708         }
 709
 710         /* Fill gaps that we discover now that we have some video which needs to be emitted.
 711            This is where we need to fill to.
 712         */
 713         DCPTime fill_to = min (time, piece->content->end());
 714
 715         if (_last_video_time) {
 716                 DCPTime fill_from = max (*_last_video_time, piece->content->position());
 717                 LastVideoMap::const_iterator last = _last_video.find (wp);
 718                 if (_film->three_d()) {
 719                         DCPTime j = fill_from;
 720                         Eyes eyes = _last_video_eyes.get_value_or(EYES_LEFT);
 721                         if (eyes == EYES_BOTH) {
 722                                 eyes = EYES_LEFT;
 723                         }
 724                         while (j < fill_to || eyes != video.eyes) {
 725                                 if (last != _last_video.end()) {
 726                                         shared_ptr<PlayerVideo> copy = last->second->shallow_copy();
 727                                         copy->set_eyes (eyes);
 728                                         emit_video (copy, j);
 729                                 } else {
 730                                         emit_video (black_player_video_frame(eyes), j);
 731                                 }
 732                                 if (eyes == EYES_RIGHT) {
 733                                         j += one_video_frame();
 734                                 }
 735                                 eyes = increment_eyes (eyes);
 736                         }
 737                 } else {
 738                         for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
 739                                 if (last != _last_video.end()) {
 740                                         emit_video (last->second, j);
 741                                 } else {
 742                                         emit_video (black_player_video_frame(EYES_BOTH), j);
 743                                 }
 744                         }
 745                 }
 746         }
 747
 748         _last_video[wp].reset (
 749                 new PlayerVideo (
 750                         video.image,
 751                         piece->content->video->crop (),
 752                         piece->content->video->fade (video.frame),
 753                         piece->content->video->scale().size (
 754                                 piece->content->video, _video_container_size, _film->frame_size ()
 755                                 ),
 756                         _video_container_size,
 757                         video.eyes,
 758                         video.part,
 759                         piece->content->video->colour_conversion(),
 760                         piece->content,
 761                         video.frame
 762                         )
 763                 );
 764
 765         DCPTime t = time;
 766         for (int i = 0; i < frc.repeat; ++i) {
 767                 if (t < piece->content->end()) {
 768                         emit_video (_last_video[wp], t);
 769                 }
 770                 t += one_video_frame ();
 771         }
 772 }
 773
 774 void
 775 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 776 {
 777         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 778
 779         shared_ptr<Piece> piece = wp.lock ();
 780         if (!piece) {
 781                 return;
 782         }
 783
 784         shared_ptr<AudioContent> content = piece->content->audio;
 785         DCPOMATIC_ASSERT (content);
 786
 787         /* Compute time in the DCP */
 788         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame);
 789         /* And the end of this block in the DCP */
 790         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), content->resampled_frame_rate());
 791
 792         /* Remove anything that comes before the start or after the end of the content */
 793         if (time < piece->content->position()) {
 794                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 795                 if (!cut.first) {
 796                         /* This audio is entirely discarded */
 797                         return;
 798                 }
 799                 content_audio.audio = cut.first;
 800                 time = cut.second;
 801         } else if (time > piece->content->end()) {
 802                 /* Discard it all */
 803                 return;
 804         } else if (end > piece->content->end()) {
 805                 Frame const remaining_frames = DCPTime(piece->content->end() - time).frames_round(_film->audio_frame_rate());
 806                 if (remaining_frames == 0) {
 807                         return;
 808                 }
 809                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 810                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 811                 content_audio.audio = cut;
 812         }
 813
 814         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 815
 816         /* Gain */
 817
 818         if (content->gain() != 0) {
 819                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 820                 gain->apply_gain (content->gain ());
 821                 content_audio.audio = gain;
 822         }
 823
 824         /* Remap */
 825
 826         content_audio.audio = remap (content_audio.audio, _film->audio_channels(), stream->mapping());
 827
 828         /* Process */
 829
 830         if (_audio_processor) {
 831                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 832         }
 833
 834         /* Push */
 835
 836         _audio_merger.push (content_audio.audio, time);
 837         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 838         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 839 }
 840
 841 void
 842 Player::image_subtitle_start (weak_ptr<Piece> wp, ContentImageSubtitle subtitle)
 843 {
 844         shared_ptr<Piece> piece = wp.lock ();
 845         if (!piece) {
 846                 return;
 847         }
 848
 849         /* Apply content's subtitle offsets */
 850         subtitle.sub.rectangle.x += piece->content->subtitle->x_offset ();
 851         subtitle.sub.rectangle.y += piece->content->subtitle->y_offset ();
 852
 853         /* Apply a corrective translation to keep the subtitle centred after the scale that is coming up */
 854         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * ((piece->content->subtitle->x_scale() - 1) / 2);
 855         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * ((piece->content->subtitle->y_scale() - 1) / 2);
 856
 857         /* Apply content's subtitle scale */
 858         subtitle.sub.rectangle.width *= piece->content->subtitle->x_scale ();
 859         subtitle.sub.rectangle.height *= piece->content->subtitle->y_scale ();
 860
 861         PlayerSubtitles ps;
 862         ps.image.push_back (subtitle.sub);
 863         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 864
 865         _active_subtitles.add_from (wp, ps, from);
 866 }
 867
 868 void
 869 Player::text_subtitle_start (weak_ptr<Piece> wp, ContentTextSubtitle subtitle)
 870 {
 871         shared_ptr<Piece> piece = wp.lock ();
 872         if (!piece) {
 873                 return;
 874         }
 875
 876         PlayerSubtitles ps;
 877         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 878
 879         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 880                 s.set_h_position (s.h_position() + piece->content->subtitle->x_offset ());
 881                 s.set_v_position (s.v_position() + piece->content->subtitle->y_offset ());
 882                 float const xs = piece->content->subtitle->x_scale();
 883                 float const ys = piece->content->subtitle->y_scale();
 884                 float size = s.size();
 885
 886                 /* Adjust size to express the common part of the scaling;
 887                    e.g. if xs = ys = 0.5 we scale size by 2.
 888                 */
 889                 if (xs > 1e-5 && ys > 1e-5) {
 890                         size *= 1 / min (1 / xs, 1 / ys);
 891                 }
 892                 s.set_size (size);
 893
 894                 /* Then express aspect ratio changes */
 895                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 896                         s.set_aspect_adjust (xs / ys);
 897                 }
 898
 899                 s.set_in (dcp::Time(from.seconds(), 1000));
 900                 ps.text.push_back (SubtitleString (s, piece->content->subtitle->outline_width()));
 901                 ps.add_fonts (piece->content->subtitle->fonts ());
 902         }
 903
 904         _active_subtitles.add_from (wp, ps, from);
 905 }
 906
 907 void
 908 Player::subtitle_stop (weak_ptr<Piece> wp, ContentTime to)
 909 {
 910         if (!_active_subtitles.have (wp)) {
 911                 return;
 912         }
 913
 914         shared_ptr<Piece> piece = wp.lock ();
 915         if (!piece) {
 916                 return;
 917         }
 918
 919         DCPTime const dcp_to = content_time_to_dcp (piece, to);
 920
 921         pair<PlayerSubtitles, DCPTime> from = _active_subtitles.add_to (wp, dcp_to);
 922
 923         if (piece->content->subtitle->use() && !_always_burn_subtitles && !piece->content->subtitle->burn()) {
 924                 Subtitle (from.first, DCPTimePeriod (from.second, dcp_to));
 925         }
 926 }
 927
 928 void
 929 Player::seek (DCPTime time, bool accurate)
 930 {
 931         if (!_have_valid_pieces) {
 932                 setup_pieces ();
 933         }
 934
 935         if (_shuffler) {
 936                 _shuffler->clear ();
 937         }
 938
 939         _delay.clear ();
 940
 941         if (_audio_processor) {
 942                 _audio_processor->flush ();
 943         }
 944
 945         _audio_merger.clear ();
 946         _active_subtitles.clear ();
 947
 948         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 949                 if (time < i->content->position()) {
 950                         /* Before; seek to 0 */
 951                         i->decoder->seek (ContentTime(), accurate);
 952                         i->done = false;
 953                 } else if (i->content->position() <= time && time < i->content->end()) {
 954                         /* During; seek to position */
 955                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
 956                         i->done = false;
 957                 } else {
 958                         /* After; this piece is done */
 959                         i->done = true;
 960                 }
 961         }
 962
 963         if (accurate) {
 964                 _last_video_time = time;
 965                 _last_video_eyes = EYES_LEFT;
 966                 _last_audio_time = time;
 967         } else {
 968                 _last_video_time = optional<DCPTime>();
 969                 _last_video_eyes = optional<Eyes>();
 970                 _last_audio_time = optional<DCPTime>();
 971         }
 972
 973         _black.set_position (time);
 974         _silent.set_position (time);
 975
 976         _last_video.clear ();
 977 }
 978
 979 void
 980 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
 981 {
 982         /* We need a delay to give a little wiggle room to ensure that relevent subtitles arrive at the
 983            player before the video that requires them.
 984         */
 985         _delay.push_back (make_pair (pv, time));
 986
 987         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
 988                 _last_video_time = time + one_video_frame();
 989         }
 990         _last_video_eyes = increment_eyes (pv->eyes());
 991
 992         if (_delay.size() < 3) {
 993                 return;
 994         }
 995
 996         pair<shared_ptr<PlayerVideo>, DCPTime> to_do = _delay.front();
 997         _delay.pop_front();
 998         do_emit_video (to_do.first, to_do.second);
 999 }
1000
1001 void
1002 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1003 {
1004         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1005                 _active_subtitles.clear_before (time);
1006         }
1007
1008         optional<PositionImage> subtitles = subtitles_for_frame (time);
1009         if (subtitles) {
1010                 pv->set_subtitle (subtitles.get ());
1011         }
1012
1013         Video (pv, time);
1014 }
1015
1016 void
1017 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1018 {
1019         /* Log if the assert below is about to fail */
1020         if (_last_audio_time && time != *_last_audio_time) {
1021                 _film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_last_audio_time)), LogEntry::TYPE_WARNING);
1022         }
1023
1024         /* This audio must follow on from the previous */
1025         DCPOMATIC_ASSERT (!_last_audio_time || time == *_last_audio_time);
1026         Audio (data, time);
1027         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate());
1028 }
1029
1030 void
1031 Player::fill_audio (DCPTimePeriod period)
1032 {
1033         if (period.from == period.to) {
1034                 return;
1035         }
1036
1037         DCPOMATIC_ASSERT (period.from < period.to);
1038
1039         DCPTime t = period.from;
1040         while (t < period.to) {
1041                 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1042                 Frame const samples = block.frames_round(_film->audio_frame_rate());
1043                 if (samples) {
1044                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1045                         silence->make_silent ();
1046                         emit_audio (silence, t);
1047                 }
1048                 t += block;
1049         }
1050 }
1051
1052 DCPTime
1053 Player::one_video_frame () const
1054 {
1055         return DCPTime::from_frames (1, _film->video_frame_rate ());
1056 }
1057
1058 pair<shared_ptr<AudioBuffers>, DCPTime>
1059 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1060 {
1061         DCPTime const discard_time = discard_to - time;
1062         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1063         Frame remaining_frames = audio->frames() - discard_frames;
1064         if (remaining_frames <= 0) {
1065                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1066         }
1067         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1068         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1069         return make_pair(cut, time + discard_time);
1070 }
1071
1072 void
1073 Player::set_dcp_decode_reduction (optional<int> reduction)
1074 {
1075         if (reduction == _dcp_decode_reduction) {
1076                 return;
1077         }
1078
1079         _dcp_decode_reduction = reduction;
1080         _have_valid_pieces = false;
1081         Changed (PlayerProperty::DCP_DECODE_REDUCTION, false);
1082 }