src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2018 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_text.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "text_content.h"
  44 #include "text_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "dcp_decoder.h"
  48 #include "image_decoder.h"
  49 #include "compose.hpp"
  50 #include "shuffler.h"
  51 #include <dcp/reel.h>
  52 #include <dcp/reel_sound_asset.h>
  53 #include <dcp/reel_subtitle_asset.h>
  54 #include <dcp/reel_picture_asset.h>
  55 #include <boost/foreach.hpp>
  56 #include <stdint.h>
  57 #include <algorithm>
  58 #include <iostream>
  59
  60 #include "i18n.h"
  61
  62 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
  63
  64 using std::list;
  65 using std::cout;
  66 using std::min;
  67 using std::max;
  68 using std::min;
  69 using std::vector;
  70 using std::pair;
  71 using std::map;
  72 using std::make_pair;
  73 using std::copy;
  74 using boost::shared_ptr;
  75 using boost::weak_ptr;
  76 using boost::dynamic_pointer_cast;
  77 using boost::optional;
  78 using boost::scoped_ptr;
  79
  80 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
  81 int const PlayerProperty::PLAYLIST = 701;
  82 int const PlayerProperty::FILM_CONTAINER = 702;
  83 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
  84 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
  85
  86 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  87         : _film (film)
  88         , _playlist (playlist)
  89         , _have_valid_pieces (false)
  90         , _ignore_video (false)
  91         , _ignore_subtitle (false)
  92         , _always_burn_subtitles (false)
  93         , _fast (false)
  94         , _play_referenced (false)
  95         , _audio_merger (_film->audio_frame_rate())
  96         , _shuffler (0)
  97 {
  98         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  99         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 100         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
 101         set_video_container_size (_film->frame_size ());
 102
 103         film_changed (Film::AUDIO_PROCESSOR);
 104
 105         seek (DCPTime (), true);
 106 }
 107
 108 Player::~Player ()
 109 {
 110         delete _shuffler;
 111 }
 112
 113 void
 114 Player::setup_pieces ()
 115 {
 116         _pieces.clear ();
 117
 118         delete _shuffler;
 119         _shuffler = new Shuffler();
 120         _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
 121
 122         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 123
 124                 if (!i->paths_valid ()) {
 125                         continue;
 126                 }
 127
 128                 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log(), _fast);
 129                 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
 130
 131                 if (!decoder) {
 132                         /* Not something that we can decode; e.g. Atmos content */
 133                         continue;
 134                 }
 135
 136                 if (decoder->video && _ignore_video) {
 137                         decoder->video->set_ignore (true);
 138                 }
 139
 140                 if (decoder->subtitle && _ignore_subtitle) {
 141                         decoder->subtitle->set_ignore (true);
 142                 }
 143
 144                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 145                 if (dcp) {
 146                         dcp->set_decode_referenced (_play_referenced);
 147                         if (_play_referenced) {
 148                                 dcp->set_forced_reduction (_dcp_decode_reduction);
 149                         }
 150                 }
 151
 152                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 153                 _pieces.push_back (piece);
 154
 155                 if (decoder->video) {
 156                         if (i->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT || i->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) {
 157                                 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
 158                                 decoder->video->Data.connect (bind (&Shuffler::video, _shuffler, weak_ptr<Piece>(piece), _1));
 159                         } else {
 160                                 decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece>(piece), _1));
 161                         }
 162                 }
 163
 164                 if (decoder->audio) {
 165                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 166                 }
 167
 168                 if (decoder->subtitle) {
 169                         decoder->subtitle->BitmapStart.connect (bind (&Player::bitmap_text_start, this, weak_ptr<Piece> (piece), _1));
 170                         decoder->subtitle->PlainStart.connect (bind (&Player::plain_text_start, this, weak_ptr<Piece> (piece), _1));
 171                         decoder->subtitle->Stop.connect (bind (&Player::subtitle_stop, this, weak_ptr<Piece> (piece), _1, _2));
 172                 }
 173         }
 174
 175         _stream_states.clear ();
 176         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 177                 if (i->content->audio) {
 178                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 179                                 _stream_states[j] = StreamState (i, i->content->position ());
 180                         }
 181                 }
 182         }
 183
 184         _black = Empty (_film->content(), _film->length(), bind(&Content::video, _1));
 185         _silent = Empty (_film->content(), _film->length(), bind(&Content::audio, _1));
 186
 187         _last_video_time = DCPTime ();
 188         _last_video_eyes = EYES_BOTH;
 189         _last_audio_time = DCPTime ();
 190         _have_valid_pieces = true;
 191 }
 192
 193 void
 194 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
 195 {
 196         shared_ptr<Content> c = w.lock ();
 197         if (!c) {
 198                 return;
 199         }
 200
 201         if (
 202                 property == ContentProperty::POSITION ||
 203                 property == ContentProperty::LENGTH ||
 204                 property == ContentProperty::TRIM_START ||
 205                 property == ContentProperty::TRIM_END ||
 206                 property == ContentProperty::PATH ||
 207                 property == VideoContentProperty::FRAME_TYPE ||
 208                 property == VideoContentProperty::COLOUR_CONVERSION ||
 209                 property == AudioContentProperty::STREAMS ||
 210                 property == DCPContentProperty::NEEDS_ASSETS ||
 211                 property == DCPContentProperty::NEEDS_KDM ||
 212                 property == TextContentProperty::COLOUR ||
 213                 property == TextContentProperty::EFFECT ||
 214                 property == TextContentProperty::EFFECT_COLOUR ||
 215                 property == FFmpegContentProperty::SUBTITLE_STREAM ||
 216                 property == FFmpegContentProperty::FILTERS
 217                 ) {
 218
 219                 _have_valid_pieces = false;
 220                 Changed (property, frequent);
 221
 222         } else if (
 223                 property == TextContentProperty::LINE_SPACING ||
 224                 property == TextContentProperty::OUTLINE_WIDTH ||
 225                 property == TextContentProperty::Y_SCALE ||
 226                 property == TextContentProperty::FADE_IN ||
 227                 property == TextContentProperty::FADE_OUT ||
 228                 property == ContentProperty::VIDEO_FRAME_RATE ||
 229                 property == TextContentProperty::USE ||
 230                 property == TextContentProperty::X_OFFSET ||
 231                 property == TextContentProperty::Y_OFFSET ||
 232                 property == TextContentProperty::X_SCALE ||
 233                 property == TextContentProperty::FONTS ||
 234                 property == VideoContentProperty::CROP ||
 235                 property == VideoContentProperty::SCALE ||
 236                 property == VideoContentProperty::FADE_IN ||
 237                 property == VideoContentProperty::FADE_OUT
 238                 ) {
 239
 240                 Changed (property, frequent);
 241         }
 242 }
 243
 244 void
 245 Player::set_video_container_size (dcp::Size s)
 246 {
 247         if (s == _video_container_size) {
 248                 return;
 249         }
 250
 251         _video_container_size = s;
 252
 253         _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 254         _black_image->make_black ();
 255
 256         Changed (PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 257 }
 258
 259 void
 260 Player::playlist_changed ()
 261 {
 262         _have_valid_pieces = false;
 263         Changed (PlayerProperty::PLAYLIST, false);
 264 }
 265
 266 void
 267 Player::film_changed (Film::Property p)
 268 {
 269         /* Here we should notice Film properties that affect our output, and
 270            alert listeners that our output now would be different to how it was
 271            last time we were run.
 272         */
 273
 274         if (p == Film::CONTAINER) {
 275                 Changed (PlayerProperty::FILM_CONTAINER, false);
 276         } else if (p == Film::VIDEO_FRAME_RATE) {
 277                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 278                    so we need new pieces here.
 279                 */
 280                 _have_valid_pieces = false;
 281                 Changed (PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
 282         } else if (p == Film::AUDIO_PROCESSOR) {
 283                 if (_film->audio_processor ()) {
 284                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 285                 }
 286         } else if (p == Film::AUDIO_CHANNELS) {
 287                 _audio_merger.clear ();
 288         }
 289 }
 290
 291 list<PositionImage>
 292 Player::transform_bitmap_texts (list<BitmapText> subs) const
 293 {
 294         list<PositionImage> all;
 295
 296         for (list<BitmapText>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 297                 if (!i->image) {
 298                         continue;
 299                 }
 300
 301                 /* We will scale the subtitle up to fit _video_container_size */
 302                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
 303
 304                 all.push_back (
 305                         PositionImage (
 306                                 i->image->scale (
 307                                         scaled_size,
 308                                         dcp::YUV_TO_RGB_REC601,
 309                                         i->image->pixel_format (),
 310                                         true,
 311                                         _fast
 312                                         ),
 313                                 Position<int> (
 314                                         lrint (_video_container_size.width * i->rectangle.x),
 315                                         lrint (_video_container_size.height * i->rectangle.y)
 316                                         )
 317                                 )
 318                         );
 319         }
 320
 321         return all;
 322 }
 323
 324 shared_ptr<PlayerVideo>
 325 Player::black_player_video_frame (Eyes eyes) const
 326 {
 327         return shared_ptr<PlayerVideo> (
 328                 new PlayerVideo (
 329                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 330                         Crop (),
 331                         optional<double> (),
 332                         _video_container_size,
 333                         _video_container_size,
 334                         eyes,
 335                         PART_WHOLE,
 336                         PresetColourConversion::all().front().conversion,
 337                         boost::weak_ptr<Content>(),
 338                         boost::optional<Frame>()
 339                 )
 340         );
 341 }
 342
 343 Frame
 344 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 345 {
 346         DCPTime s = t - piece->content->position ();
 347         s = min (piece->content->length_after_trim(), s);
 348         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 349
 350         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 351            then convert that ContentTime to frames at the content's rate.  However this fails for
 352            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 353            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 354
 355            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 356         */
 357         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 358 }
 359
 360 DCPTime
 361 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 362 {
 363         /* See comment in dcp_to_content_video */
 364         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime(piece->content->trim_start(), piece->frc);
 365         return d + piece->content->position();
 366 }
 367
 368 Frame
 369 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 370 {
 371         DCPTime s = t - piece->content->position ();
 372         s = min (piece->content->length_after_trim(), s);
 373         /* See notes in dcp_to_content_video */
 374         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 375 }
 376
 377 DCPTime
 378 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 379 {
 380         /* See comment in dcp_to_content_video */
 381         return DCPTime::from_frames (f, _film->audio_frame_rate())
 382                 - DCPTime (piece->content->trim_start(), piece->frc)
 383                 + piece->content->position();
 384 }
 385
 386 ContentTime
 387 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 388 {
 389         DCPTime s = t - piece->content->position ();
 390         s = min (piece->content->length_after_trim(), s);
 391         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 392 }
 393
 394 DCPTime
 395 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 396 {
 397         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 398 }
 399
 400 list<shared_ptr<Font> >
 401 Player::get_subtitle_fonts ()
 402 {
 403         if (!_have_valid_pieces) {
 404                 setup_pieces ();
 405         }
 406
 407         list<shared_ptr<Font> > fonts;
 408         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
 409                 if (p->content->subtitle) {
 410                         /* XXX: things may go wrong if there are duplicate font IDs
 411                            with different font files.
 412                         */
 413                         list<shared_ptr<Font> > f = p->content->subtitle->fonts ();
 414                         copy (f.begin(), f.end(), back_inserter (fonts));
 415                 }
 416         }
 417
 418         return fonts;
 419 }
 420
 421 /** Set this player never to produce any video data */
 422 void
 423 Player::set_ignore_video ()
 424 {
 425         _ignore_video = true;
 426 }
 427
 428 void
 429 Player::set_ignore_subtitle ()
 430 {
 431         _ignore_subtitle = true;
 432 }
 433
 434 /** Set whether or not this player should always burn text subtitles into the image,
 435  *  regardless of the content settings.
 436  *  @param burn true to always burn subtitles, false to obey content settings.
 437  */
 438 void
 439 Player::set_always_burn_subtitles (bool burn)
 440 {
 441         _always_burn_subtitles = burn;
 442 }
 443
 444 /** Sets up the player to be faster, possibly at the expense of quality */
 445 void
 446 Player::set_fast ()
 447 {
 448         _fast = true;
 449         _have_valid_pieces = false;
 450 }
 451
 452 void
 453 Player::set_play_referenced ()
 454 {
 455         _play_referenced = true;
 456         _have_valid_pieces = false;
 457 }
 458
 459 list<ReferencedReelAsset>
 460 Player::get_reel_assets ()
 461 {
 462         list<ReferencedReelAsset> a;
 463
 464         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 465                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 466                 if (!j) {
 467                         continue;
 468                 }
 469
 470                 scoped_ptr<DCPDecoder> decoder;
 471                 try {
 472                         decoder.reset (new DCPDecoder (j, _film->log(), false));
 473                 } catch (...) {
 474                         return a;
 475                 }
 476
 477                 int64_t offset = 0;
 478                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 479
 480                         DCPOMATIC_ASSERT (j->video_frame_rate ());
 481                         double const cfr = j->video_frame_rate().get();
 482                         Frame const trim_start = j->trim_start().frames_round (cfr);
 483                         Frame const trim_end = j->trim_end().frames_round (cfr);
 484                         int const ffr = _film->video_frame_rate ();
 485
 486                         DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
 487                         if (j->reference_video ()) {
 488                                 shared_ptr<dcp::ReelAsset> ra = k->main_picture ();
 489                                 DCPOMATIC_ASSERT (ra);
 490                                 ra->set_entry_point (ra->entry_point() + trim_start);
 491                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 492                                 a.push_back (
 493                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 494                                         );
 495                         }
 496
 497                         if (j->reference_audio ()) {
 498                                 shared_ptr<dcp::ReelAsset> ra = k->main_sound ();
 499                                 DCPOMATIC_ASSERT (ra);
 500                                 ra->set_entry_point (ra->entry_point() + trim_start);
 501                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 502                                 a.push_back (
 503                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 504                                         );
 505                         }
 506
 507                         if (j->reference_subtitle ()) {
 508                                 shared_ptr<dcp::ReelAsset> ra = k->main_subtitle ();
 509                                 DCPOMATIC_ASSERT (ra);
 510                                 ra->set_entry_point (ra->entry_point() + trim_start);
 511                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 512                                 a.push_back (
 513                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 514                                         );
 515                         }
 516
 517                         /* Assume that main picture duration is the length of the reel */
 518                         offset += k->main_picture()->duration ();
 519                 }
 520         }
 521
 522         return a;
 523 }
 524
 525 bool
 526 Player::pass ()
 527 {
 528         if (!_have_valid_pieces) {
 529                 setup_pieces ();
 530         }
 531
 532         if (_playlist->length() == DCPTime()) {
 533                 /* Special case of an empty Film; just give one black frame */
 534                 emit_video (black_player_video_frame(EYES_BOTH), DCPTime());
 535                 return true;
 536         }
 537
 538         /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
 539
 540         shared_ptr<Piece> earliest_content;
 541         optional<DCPTime> earliest_time;
 542
 543         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 544                 if (i->done) {
 545                         continue;
 546                 }
 547
 548                 DCPTime const t = content_time_to_dcp (i, max(i->decoder->position(), i->content->trim_start()));
 549                 if (t > i->content->end()) {
 550                         i->done = true;
 551                 } else {
 552
 553                         /* Given two choices at the same time, pick the one with a subtitle so we see it before
 554                            the video.
 555                         */
 556                         if (!earliest_time || t < *earliest_time || (t == *earliest_time && i->decoder->subtitle)) {
 557                                 earliest_time = t;
 558                                 earliest_content = i;
 559                         }
 560                 }
 561         }
 562
 563         bool done = false;
 564
 565         enum {
 566                 NONE,
 567                 CONTENT,
 568                 BLACK,
 569                 SILENT
 570         } which = NONE;
 571
 572         if (earliest_content) {
 573                 which = CONTENT;
 574         }
 575
 576         if (!_black.done() && (!earliest_time || _black.position() < *earliest_time)) {
 577                 earliest_time = _black.position ();
 578                 which = BLACK;
 579         }
 580
 581         if (!_silent.done() && (!earliest_time || _silent.position() < *earliest_time)) {
 582                 earliest_time = _silent.position ();
 583                 which = SILENT;
 584         }
 585
 586         switch (which) {
 587         case CONTENT:
 588                 earliest_content->done = earliest_content->decoder->pass ();
 589                 break;
 590         case BLACK:
 591                 emit_video (black_player_video_frame(EYES_BOTH), _black.position());
 592                 _black.set_position (_black.position() + one_video_frame());
 593                 break;
 594         case SILENT:
 595         {
 596                 DCPTimePeriod period (_silent.period_at_position());
 597                 if (_last_audio_time) {
 598                         /* Sometimes the thing that happened last finishes fractionally before
 599                            this silence.  Bodge the start time of the silence to fix it.  I'm
 600                            not sure if this is the right solution --- maybe the last thing should
 601                            be padded `forward' rather than this thing padding `back'.
 602                         */
 603                         period.from = min(period.from, *_last_audio_time);
 604                 }
 605                 if (period.duration() > one_video_frame()) {
 606                         period.to = period.from + one_video_frame();
 607                 }
 608                 fill_audio (period);
 609                 _silent.set_position (period.to);
 610                 break;
 611         }
 612         case NONE:
 613                 done = true;
 614                 break;
 615         }
 616
 617         /* Emit any audio that is ready */
 618
 619         /* Work out the time before which the audio is definitely all here.  This is the earliest last_push_end of one
 620            of our streams, or the position of the _silent.
 621         */
 622         DCPTime pull_to = _film->length ();
 623         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 624                 if (!i->second.piece->done && i->second.last_push_end < pull_to) {
 625                         pull_to = i->second.last_push_end;
 626                 }
 627         }
 628         if (!_silent.done() && _silent.position() < pull_to) {
 629                 pull_to = _silent.position();
 630         }
 631
 632         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 633         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 634                 if (_last_audio_time && i->second < *_last_audio_time) {
 635                         /* This new data comes before the last we emitted (or the last seek); discard it */
 636                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 637                         if (!cut.first) {
 638                                 continue;
 639                         }
 640                         *i = cut;
 641                 } else if (_last_audio_time && i->second > *_last_audio_time) {
 642                         /* There's a gap between this data and the last we emitted; fill with silence */
 643                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 644                 }
 645
 646                 emit_audio (i->first, i->second);
 647         }
 648
 649         if (done) {
 650                 _shuffler->flush ();
 651                 for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 652                         do_emit_video(i->first, i->second);
 653                 }
 654         }
 655
 656         return done;
 657 }
 658
 659 optional<PositionImage>
 660 Player::subtitles_for_frame (DCPTime time) const
 661 {
 662         list<PositionImage> subtitles;
 663
 664         int const vfr = _film->video_frame_rate();
 665
 666         BOOST_FOREACH (PlayerCaption i, _active_text[TEXT_SUBTITLE].get_burnt (DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), _always_burn_subtitles)) {
 667
 668                 /* Image subtitles */
 669                 list<PositionImage> c = transform_bitmap_texts (i.image);
 670                 copy (c.begin(), c.end(), back_inserter (subtitles));
 671
 672                 /* Text subtitles (rendered to an image) */
 673                 if (!i.text.empty ()) {
 674                         list<PositionImage> s = render_text (i.text, i.fonts, _video_container_size, time, vfr);
 675                         copy (s.begin(), s.end(), back_inserter (subtitles));
 676                 }
 677         }
 678
 679         if (subtitles.empty ()) {
 680                 return optional<PositionImage> ();
 681         }
 682
 683         return merge (subtitles);
 684 }
 685
 686 void
 687 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 688 {
 689         shared_ptr<Piece> piece = wp.lock ();
 690         if (!piece) {
 691                 return;
 692         }
 693
 694         FrameRateChange frc(piece->content->active_video_frame_rate(), _film->video_frame_rate());
 695         if (frc.skip && (video.frame % 2) == 1) {
 696                 return;
 697         }
 698
 699         /* Time of the first frame we will emit */
 700         DCPTime const time = content_video_to_dcp (piece, video.frame);
 701
 702         /* Discard if it's before the content's period or the last accurate seek.  We can't discard
 703            if it's after the content's period here as in that case we still need to fill any gap between
 704            `now' and the end of the content's period.
 705         */
 706         if (time < piece->content->position() || (_last_video_time && time < *_last_video_time)) {
 707                 return;
 708         }
 709
 710         /* Fill gaps that we discover now that we have some video which needs to be emitted.
 711            This is where we need to fill to.
 712         */
 713         DCPTime fill_to = min (time, piece->content->end());
 714
 715         if (_last_video_time) {
 716                 DCPTime fill_from = max (*_last_video_time, piece->content->position());
 717                 LastVideoMap::const_iterator last = _last_video.find (wp);
 718                 if (_film->three_d()) {
 719                         DCPTime j = fill_from;
 720                         Eyes eyes = _last_video_eyes.get_value_or(EYES_LEFT);
 721                         if (eyes == EYES_BOTH) {
 722                                 eyes = EYES_LEFT;
 723                         }
 724                         while (j < fill_to || eyes != video.eyes) {
 725                                 if (last != _last_video.end()) {
 726                                         shared_ptr<PlayerVideo> copy = last->second->shallow_copy();
 727                                         copy->set_eyes (eyes);
 728                                         emit_video (copy, j);
 729                                 } else {
 730                                         emit_video (black_player_video_frame(eyes), j);
 731                                 }
 732                                 if (eyes == EYES_RIGHT) {
 733                                         j += one_video_frame();
 734                                 }
 735                                 eyes = increment_eyes (eyes);
 736                         }
 737                 } else {
 738                         for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
 739                                 if (last != _last_video.end()) {
 740                                         emit_video (last->second, j);
 741                                 } else {
 742                                         emit_video (black_player_video_frame(EYES_BOTH), j);
 743                                 }
 744                         }
 745                 }
 746         }
 747
 748         _last_video[wp].reset (
 749                 new PlayerVideo (
 750                         video.image,
 751                         piece->content->video->crop (),
 752                         piece->content->video->fade (video.frame),
 753                         piece->content->video->scale().size (
 754                                 piece->content->video, _video_container_size, _film->frame_size ()
 755                                 ),
 756                         _video_container_size,
 757                         video.eyes,
 758                         video.part,
 759                         piece->content->video->colour_conversion(),
 760                         piece->content,
 761                         video.frame
 762                         )
 763                 );
 764
 765         DCPTime t = time;
 766         for (int i = 0; i < frc.repeat; ++i) {
 767                 if (t < piece->content->end()) {
 768                         emit_video (_last_video[wp], t);
 769                 }
 770                 t += one_video_frame ();
 771         }
 772 }
 773
 774 void
 775 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 776 {
 777         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 778
 779         shared_ptr<Piece> piece = wp.lock ();
 780         if (!piece) {
 781                 return;
 782         }
 783
 784         shared_ptr<AudioContent> content = piece->content->audio;
 785         DCPOMATIC_ASSERT (content);
 786
 787         /* Compute time in the DCP */
 788         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame);
 789         /* And the end of this block in the DCP */
 790         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), content->resampled_frame_rate());
 791
 792         /* Remove anything that comes before the start or after the end of the content */
 793         if (time < piece->content->position()) {
 794                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 795                 if (!cut.first) {
 796                         /* This audio is entirely discarded */
 797                         return;
 798                 }
 799                 content_audio.audio = cut.first;
 800                 time = cut.second;
 801         } else if (time > piece->content->end()) {
 802                 /* Discard it all */
 803                 return;
 804         } else if (end > piece->content->end()) {
 805                 Frame const remaining_frames = DCPTime(piece->content->end() - time).frames_round(_film->audio_frame_rate());
 806                 if (remaining_frames == 0) {
 807                         return;
 808                 }
 809                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 810                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 811                 content_audio.audio = cut;
 812         }
 813
 814         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 815
 816         /* Gain */
 817
 818         if (content->gain() != 0) {
 819                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 820                 gain->apply_gain (content->gain ());
 821                 content_audio.audio = gain;
 822         }
 823
 824         /* Remap */
 825
 826         content_audio.audio = remap (content_audio.audio, _film->audio_channels(), stream->mapping());
 827
 828         /* Process */
 829
 830         if (_audio_processor) {
 831                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 832         }
 833
 834         /* Push */
 835
 836         _audio_merger.push (content_audio.audio, time);
 837         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 838         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 839 }
 840
 841 void
 842 Player::bitmap_text_start (weak_ptr<Piece> wp, ContentBitmapCaption subtitle)
 843 {
 844         shared_ptr<Piece> piece = wp.lock ();
 845         if (!piece) {
 846                 return;
 847         }
 848
 849         /* Apply content's subtitle offsets */
 850         subtitle.sub.rectangle.x += piece->content->subtitle->x_offset ();
 851         subtitle.sub.rectangle.y += piece->content->subtitle->y_offset ();
 852
 853         /* Apply a corrective translation to keep the subtitle centred after the scale that is coming up */
 854         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * ((piece->content->subtitle->x_scale() - 1) / 2);
 855         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * ((piece->content->subtitle->y_scale() - 1) / 2);
 856
 857         /* Apply content's subtitle scale */
 858         subtitle.sub.rectangle.width *= piece->content->subtitle->x_scale ();
 859         subtitle.sub.rectangle.height *= piece->content->subtitle->y_scale ();
 860
 861         PlayerCaption ps;
 862         ps.image.push_back (subtitle.sub);
 863         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 864
 865         _active_text[subtitle.type()].add_from (wp, ps, from);
 866 }
 867
 868 void
 869 Player::plain_text_start (weak_ptr<Piece> wp, ContentTextCaption subtitle)
 870 {
 871         shared_ptr<Piece> piece = wp.lock ();
 872         if (!piece) {
 873                 return;
 874         }
 875
 876         PlayerCaption ps;
 877         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 878
 879         if (from > piece->content->end()) {
 880                 return;
 881         }
 882
 883         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 884                 s.set_h_position (s.h_position() + piece->content->subtitle->x_offset ());
 885                 s.set_v_position (s.v_position() + piece->content->subtitle->y_offset ());
 886                 float const xs = piece->content->subtitle->x_scale();
 887                 float const ys = piece->content->subtitle->y_scale();
 888                 float size = s.size();
 889
 890                 /* Adjust size to express the common part of the scaling;
 891                    e.g. if xs = ys = 0.5 we scale size by 2.
 892                 */
 893                 if (xs > 1e-5 && ys > 1e-5) {
 894                         size *= 1 / min (1 / xs, 1 / ys);
 895                 }
 896                 s.set_size (size);
 897
 898                 /* Then express aspect ratio changes */
 899                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 900                         s.set_aspect_adjust (xs / ys);
 901                 }
 902
 903                 s.set_in (dcp::Time(from.seconds(), 1000));
 904                 ps.text.push_back (TextCaption (s, piece->content->subtitle->outline_width()));
 905                 ps.add_fonts (piece->content->subtitle->fonts ());
 906         }
 907
 908         _active_text[subtitle.type()].add_from (wp, ps, from);
 909 }
 910
 911 void
 912 Player::subtitle_stop (weak_ptr<Piece> wp, ContentTime to, TextType type)
 913 {
 914         if (!_active_text[type].have (wp)) {
 915                 return;
 916         }
 917
 918         shared_ptr<Piece> piece = wp.lock ();
 919         if (!piece) {
 920                 return;
 921         }
 922
 923         DCPTime const dcp_to = content_time_to_dcp (piece, to);
 924
 925         if (dcp_to > piece->content->end()) {
 926                 return;
 927         }
 928
 929         pair<PlayerCaption, DCPTime> from = _active_text[type].add_to (wp, dcp_to);
 930
 931         if (piece->content->subtitle->use() && !_always_burn_subtitles && !piece->content->subtitle->burn()) {
 932                 Text (from.first, type, DCPTimePeriod (from.second, dcp_to));
 933         }
 934 }
 935
 936 void
 937 Player::seek (DCPTime time, bool accurate)
 938 {
 939         if (!_have_valid_pieces) {
 940                 setup_pieces ();
 941         }
 942
 943         if (_shuffler) {
 944                 _shuffler->clear ();
 945         }
 946
 947         _delay.clear ();
 948
 949         if (_audio_processor) {
 950                 _audio_processor->flush ();
 951         }
 952
 953         _audio_merger.clear ();
 954         for (int i = 0; i < TEXT_COUNT; ++i) {
 955                 _active_text[i].clear ();
 956         }
 957
 958         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 959                 if (time < i->content->position()) {
 960                         /* Before; seek to the start of the content */
 961                         i->decoder->seek (dcp_to_content_time (i, i->content->position()), accurate);
 962                         i->done = false;
 963                 } else if (i->content->position() <= time && time < i->content->end()) {
 964                         /* During; seek to position */
 965                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
 966                         i->done = false;
 967                 } else {
 968                         /* After; this piece is done */
 969                         i->done = true;
 970                 }
 971         }
 972
 973         if (accurate) {
 974                 _last_video_time = time;
 975                 _last_video_eyes = EYES_LEFT;
 976                 _last_audio_time = time;
 977         } else {
 978                 _last_video_time = optional<DCPTime>();
 979                 _last_video_eyes = optional<Eyes>();
 980                 _last_audio_time = optional<DCPTime>();
 981         }
 982
 983         _black.set_position (time);
 984         _silent.set_position (time);
 985
 986         _last_video.clear ();
 987 }
 988
 989 void
 990 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
 991 {
 992         /* We need a delay to give a little wiggle room to ensure that relevent subtitles arrive at the
 993            player before the video that requires them.
 994         */
 995         _delay.push_back (make_pair (pv, time));
 996
 997         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
 998                 _last_video_time = time + one_video_frame();
 999         }
1000         _last_video_eyes = increment_eyes (pv->eyes());
1001
1002         if (_delay.size() < 3) {
1003                 return;
1004         }
1005
1006         pair<shared_ptr<PlayerVideo>, DCPTime> to_do = _delay.front();
1007         _delay.pop_front();
1008         do_emit_video (to_do.first, to_do.second);
1009 }
1010
1011 void
1012 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1013 {
1014         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1015                 for (int i = 0; i < TEXT_COUNT; ++i) {
1016                         _active_text[i].clear_before (time);
1017                 }
1018         }
1019
1020         optional<PositionImage> subtitles = subtitles_for_frame (time);
1021         if (subtitles) {
1022                 pv->set_subtitle (subtitles.get ());
1023         }
1024
1025         Video (pv, time);
1026 }
1027
1028 void
1029 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1030 {
1031         /* Log if the assert below is about to fail */
1032         if (_last_audio_time && time != *_last_audio_time) {
1033                 _film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_last_audio_time)), LogEntry::TYPE_WARNING);
1034         }
1035
1036         /* This audio must follow on from the previous */
1037         DCPOMATIC_ASSERT (!_last_audio_time || time == *_last_audio_time);
1038         Audio (data, time);
1039         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate());
1040 }
1041
1042 void
1043 Player::fill_audio (DCPTimePeriod period)
1044 {
1045         if (period.from == period.to) {
1046                 return;
1047         }
1048
1049         DCPOMATIC_ASSERT (period.from < period.to);
1050
1051         DCPTime t = period.from;
1052         while (t < period.to) {
1053                 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1054                 Frame const samples = block.frames_round(_film->audio_frame_rate());
1055                 if (samples) {
1056                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1057                         silence->make_silent ();
1058                         emit_audio (silence, t);
1059                 }
1060                 t += block;
1061         }
1062 }
1063
1064 DCPTime
1065 Player::one_video_frame () const
1066 {
1067         return DCPTime::from_frames (1, _film->video_frame_rate ());
1068 }
1069
1070 pair<shared_ptr<AudioBuffers>, DCPTime>
1071 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1072 {
1073         DCPTime const discard_time = discard_to - time;
1074         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1075         Frame remaining_frames = audio->frames() - discard_frames;
1076         if (remaining_frames <= 0) {
1077                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1078         }
1079         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1080         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1081         return make_pair(cut, time + discard_time);
1082 }
1083
1084 void
1085 Player::set_dcp_decode_reduction (optional<int> reduction)
1086 {
1087         if (reduction == _dcp_decode_reduction) {
1088                 return;
1089         }
1090
1091         _dcp_decode_reduction = reduction;
1092         _have_valid_pieces = false;
1093         Changed (PlayerProperty::DCP_DECODE_REDUCTION, false);
1094 }
1095
1096 DCPTime
1097 Player::content_time_to_dcp (shared_ptr<Content> content, ContentTime t)
1098 {
1099         if (_have_valid_pieces) {
1100                 setup_pieces ();
1101         }
1102
1103         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1104                 if (i->content == content) {
1105                         return content_time_to_dcp (i, t);
1106                 }
1107         }
1108
1109         DCPOMATIC_ASSERT (false);
1110         return DCPTime ();
1111 }