src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2018 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_text.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "text_content.h"
  44 #include "text_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "dcp_decoder.h"
  48 #include "image_decoder.h"
  49 #include "compose.hpp"
  50 #include "shuffler.h"
  51 #include "timer.h"
  52 #include <dcp/reel.h>
  53 #include <dcp/reel_sound_asset.h>
  54 #include <dcp/reel_subtitle_asset.h>
  55 #include <dcp/reel_picture_asset.h>
  56 #include <dcp/reel_closed_caption_asset.h>
  57 #include <boost/foreach.hpp>
  58 #include <stdint.h>
  59 #include <algorithm>
  60 #include <iostream>
  61
  62 #include "i18n.h"
  63
  64 using std::list;
  65 using std::cout;
  66 using std::min;
  67 using std::max;
  68 using std::min;
  69 using std::vector;
  70 using std::pair;
  71 using std::map;
  72 using std::make_pair;
  73 using std::copy;
  74 using boost::shared_ptr;
  75 using boost::weak_ptr;
  76 using boost::dynamic_pointer_cast;
  77 using boost::optional;
  78 using boost::scoped_ptr;
  79 using namespace dcpomatic;
  80
  81 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
  82 int const PlayerProperty::PLAYLIST = 701;
  83 int const PlayerProperty::FILM_CONTAINER = 702;
  84 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
  85 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
  86
  87 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  88         : _film (film)
  89         , _playlist (playlist)
  90         , _suspended (false)
  91         , _ignore_video (false)
  92         , _ignore_audio (false)
  93         , _ignore_text (false)
  94         , _always_burn_open_subtitles (false)
  95         , _fast (false)
  96         , _play_referenced (false)
  97         , _audio_merger (_film->audio_frame_rate())
  98         , _shuffler (0)
  99 {
 100         _film_changed_connection = _film->Change.connect (bind (&Player::film_change, this, _1, _2));
 101         /* The butler must hear about this first, so since we are proxying this through to the butler we must
 102            be first.
 103         */
 104         _playlist_change_connection = _playlist->Change.connect (bind (&Player::playlist_change, this, _1), boost::signals2::at_front);
 105         _playlist_content_change_connection = _playlist->ContentChange.connect (bind(&Player::playlist_content_change, this, _1, _3, _4));
 106         set_video_container_size (_film->frame_size ());
 107
 108         film_change (CHANGE_TYPE_DONE, Film::AUDIO_PROCESSOR);
 109
 110         setup_pieces ();
 111         seek (DCPTime (), true);
 112 }
 113
 114 Player::~Player ()
 115 {
 116         delete _shuffler;
 117 }
 118
 119 void
 120 Player::setup_pieces ()
 121 {
 122         boost::mutex::scoped_lock lm (_mutex);
 123         setup_pieces_unlocked ();
 124 }
 125
 126 bool
 127 have_video (shared_ptr<Piece> piece)
 128 {
 129         return piece->decoder && piece->decoder->video;
 130 }
 131
 132 bool
 133 have_audio (shared_ptr<Piece> piece)
 134 {
 135         return piece->decoder && piece->decoder->audio;
 136 }
 137
 138 void
 139 Player::setup_pieces_unlocked ()
 140 {
 141         list<shared_ptr<Piece> > old_pieces = _pieces;
 142         _pieces.clear ();
 143
 144         delete _shuffler;
 145         _shuffler = new Shuffler();
 146         _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
 147
 148         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 149
 150                 if (!i->paths_valid ()) {
 151                         continue;
 152                 }
 153
 154                 if (_ignore_video && _ignore_audio && i->text.empty()) {
 155                         /* We're only interested in text and this content has none */
 156                         continue;
 157                 }
 158
 159                 shared_ptr<Decoder> old_decoder;
 160                 BOOST_FOREACH (shared_ptr<Piece> j, old_pieces) {
 161                         if (j->content == i) {
 162                                 old_decoder = j->decoder;
 163                                 break;
 164                         }
 165                 }
 166
 167                 shared_ptr<Decoder> decoder = decoder_factory (_film, i, _fast, old_decoder);
 168                 FrameRateChange frc (_film, i);
 169
 170                 if (!decoder) {
 171                         /* Not something that we can decode; e.g. Atmos content */
 172                         continue;
 173                 }
 174
 175                 if (decoder->video && _ignore_video) {
 176                         decoder->video->set_ignore (true);
 177                 }
 178
 179                 if (decoder->audio && _ignore_audio) {
 180                         decoder->audio->set_ignore (true);
 181                 }
 182
 183                 if (_ignore_text) {
 184                         BOOST_FOREACH (shared_ptr<TextDecoder> i, decoder->text) {
 185                                 i->set_ignore (true);
 186                         }
 187                 }
 188
 189                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 190                 if (dcp) {
 191                         dcp->set_decode_referenced (_play_referenced);
 192                         if (_play_referenced) {
 193                                 dcp->set_forced_reduction (_dcp_decode_reduction);
 194                         }
 195                 }
 196
 197                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 198                 _pieces.push_back (piece);
 199
 200                 if (decoder->video) {
 201                         if (i->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT || i->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) {
 202                                 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
 203                                 decoder->video->Data.connect (bind (&Shuffler::video, _shuffler, weak_ptr<Piece>(piece), _1));
 204                         } else {
 205                                 decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece>(piece), _1));
 206                         }
 207                 }
 208
 209                 if (decoder->audio) {
 210                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 211                 }
 212
 213                 list<shared_ptr<TextDecoder> >::const_iterator j = decoder->text.begin();
 214
 215                 while (j != decoder->text.end()) {
 216                         (*j)->BitmapStart.connect (
 217                                 bind(&Player::bitmap_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 218                                 );
 219                         (*j)->PlainStart.connect (
 220                                 bind(&Player::plain_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 221                                 );
 222                         (*j)->Stop.connect (
 223                                 bind(&Player::subtitle_stop, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 224                                 );
 225
 226                         ++j;
 227                 }
 228         }
 229
 230         _stream_states.clear ();
 231         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 232                 if (i->content->audio) {
 233                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 234                                 _stream_states[j] = StreamState (i, i->content->position ());
 235                         }
 236                 }
 237         }
 238
 239         _black = Empty (_film, _pieces, bind(&have_video, _1));
 240         _silent = Empty (_film, _pieces, bind(&have_audio, _1));
 241
 242         _last_video_time = DCPTime ();
 243         _last_video_eyes = EYES_BOTH;
 244         _last_audio_time = DCPTime ();
 245 }
 246
 247 void
 248 Player::playlist_content_change (ChangeType type, int property, bool frequent)
 249 {
 250         if (type == CHANGE_TYPE_PENDING) {
 251                 boost::mutex::scoped_lock lm (_mutex);
 252                 /* The player content is probably about to change, so we can't carry on
 253                    until that has happened and we've rebuilt our pieces.  Stop pass()
 254                    and seek() from working until then.
 255                 */
 256                 _suspended = true;
 257         } else if (type == CHANGE_TYPE_DONE) {
 258                 /* A change in our content has gone through.  Re-build our pieces. */
 259                 setup_pieces ();
 260                 _suspended = false;
 261         } else if (type == CHANGE_TYPE_CANCELLED) {
 262                 boost::mutex::scoped_lock lm (_mutex);
 263                 _suspended = false;
 264         }
 265
 266         Change (type, property, frequent);
 267 }
 268
 269 void
 270 Player::set_video_container_size (dcp::Size s)
 271 {
 272         Change (CHANGE_TYPE_PENDING, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 273
 274         {
 275                 boost::mutex::scoped_lock lm (_mutex);
 276
 277                 if (s == _video_container_size) {
 278                         lm.unlock ();
 279                         Change (CHANGE_TYPE_CANCELLED, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 280                         return;
 281                 }
 282
 283                 _video_container_size = s;
 284
 285                 _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 286                 _black_image->make_black ();
 287         }
 288
 289         Change (CHANGE_TYPE_DONE, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 290 }
 291
 292 void
 293 Player::playlist_change (ChangeType type)
 294 {
 295         if (type == CHANGE_TYPE_DONE) {
 296                 setup_pieces ();
 297         }
 298         Change (type, PlayerProperty::PLAYLIST, false);
 299 }
 300
 301 void
 302 Player::film_change (ChangeType type, Film::Property p)
 303 {
 304         /* Here we should notice Film properties that affect our output, and
 305            alert listeners that our output now would be different to how it was
 306            last time we were run.
 307         */
 308
 309         if (p == Film::CONTAINER) {
 310                 Change (type, PlayerProperty::FILM_CONTAINER, false);
 311         } else if (p == Film::VIDEO_FRAME_RATE) {
 312                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 313                    so we need new pieces here.
 314                 */
 315                 if (type == CHANGE_TYPE_DONE) {
 316                         setup_pieces ();
 317                 }
 318                 Change (type, PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
 319         } else if (p == Film::AUDIO_PROCESSOR) {
 320                 if (type == CHANGE_TYPE_DONE && _film->audio_processor ()) {
 321                         boost::mutex::scoped_lock lm (_mutex);
 322                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 323                 }
 324         } else if (p == Film::AUDIO_CHANNELS) {
 325                 if (type == CHANGE_TYPE_DONE) {
 326                         boost::mutex::scoped_lock lm (_mutex);
 327                         _audio_merger.clear ();
 328                 }
 329         }
 330 }
 331
 332 shared_ptr<PlayerVideo>
 333 Player::black_player_video_frame (Eyes eyes) const
 334 {
 335         return shared_ptr<PlayerVideo> (
 336                 new PlayerVideo (
 337                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 338                         Crop (),
 339                         optional<double> (),
 340                         _video_container_size,
 341                         _video_container_size,
 342                         eyes,
 343                         PART_WHOLE,
 344                         PresetColourConversion::all().front().conversion,
 345                         VIDEO_RANGE_FULL,
 346                         boost::weak_ptr<Content>(),
 347                         boost::optional<Frame>()
 348                 )
 349         );
 350 }
 351
 352 Frame
 353 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 354 {
 355         DCPTime s = t - piece->content->position ();
 356         s = min (piece->content->length_after_trim(_film), s);
 357         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 358
 359         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 360            then convert that ContentTime to frames at the content's rate.  However this fails for
 361            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 362            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 363
 364            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 365         */
 366         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 367 }
 368
 369 DCPTime
 370 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 371 {
 372         /* See comment in dcp_to_content_video */
 373         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime(piece->content->trim_start(), piece->frc);
 374         return d + piece->content->position();
 375 }
 376
 377 Frame
 378 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 379 {
 380         DCPTime s = t - piece->content->position ();
 381         s = min (piece->content->length_after_trim(_film), s);
 382         /* See notes in dcp_to_content_video */
 383         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 384 }
 385
 386 DCPTime
 387 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 388 {
 389         /* See comment in dcp_to_content_video */
 390         return DCPTime::from_frames (f, _film->audio_frame_rate())
 391                 - DCPTime (piece->content->trim_start(), piece->frc)
 392                 + piece->content->position();
 393 }
 394
 395 ContentTime
 396 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 397 {
 398         DCPTime s = t - piece->content->position ();
 399         s = min (piece->content->length_after_trim(_film), s);
 400         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 401 }
 402
 403 DCPTime
 404 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 405 {
 406         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 407 }
 408
 409 list<shared_ptr<Font> >
 410 Player::get_subtitle_fonts ()
 411 {
 412         boost::mutex::scoped_lock lm (_mutex);
 413
 414         list<shared_ptr<Font> > fonts;
 415         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 416                 BOOST_FOREACH (shared_ptr<TextContent> j, i->content->text) {
 417                         /* XXX: things may go wrong if there are duplicate font IDs
 418                            with different font files.
 419                         */
 420                         list<shared_ptr<Font> > f = j->fonts ();
 421                         copy (f.begin(), f.end(), back_inserter (fonts));
 422                 }
 423         }
 424
 425         return fonts;
 426 }
 427
 428 /** Set this player never to produce any video data */
 429 void
 430 Player::set_ignore_video ()
 431 {
 432         boost::mutex::scoped_lock lm (_mutex);
 433         _ignore_video = true;
 434         setup_pieces_unlocked ();
 435 }
 436
 437 void
 438 Player::set_ignore_audio ()
 439 {
 440         boost::mutex::scoped_lock lm (_mutex);
 441         _ignore_audio = true;
 442         setup_pieces_unlocked ();
 443 }
 444
 445 void
 446 Player::set_ignore_text ()
 447 {
 448         boost::mutex::scoped_lock lm (_mutex);
 449         _ignore_text = true;
 450         setup_pieces_unlocked ();
 451 }
 452
 453 /** Set the player to always burn open texts into the image regardless of the content settings */
 454 void
 455 Player::set_always_burn_open_subtitles ()
 456 {
 457         boost::mutex::scoped_lock lm (_mutex);
 458         _always_burn_open_subtitles = true;
 459 }
 460
 461 /** Sets up the player to be faster, possibly at the expense of quality */
 462 void
 463 Player::set_fast ()
 464 {
 465         boost::mutex::scoped_lock lm (_mutex);
 466         _fast = true;
 467         setup_pieces_unlocked ();
 468 }
 469
 470 void
 471 Player::set_play_referenced ()
 472 {
 473         boost::mutex::scoped_lock lm (_mutex);
 474         _play_referenced = true;
 475         setup_pieces_unlocked ();
 476 }
 477
 478 static void
 479 maybe_add_asset (list<ReferencedReelAsset>& a, shared_ptr<dcp::ReelAsset> r, Frame reel_trim_start, Frame reel_trim_end, DCPTime from, int const ffr)
 480 {
 481         DCPOMATIC_ASSERT (r);
 482         r->set_entry_point (r->entry_point().get_value_or(0) + reel_trim_start);
 483         r->set_duration (r->actual_duration() - reel_trim_start - reel_trim_end);
 484         if (r->actual_duration() > 0) {
 485                 a.push_back (
 486                         ReferencedReelAsset(r, DCPTimePeriod(from, from + DCPTime::from_frames(r->actual_duration(), ffr)))
 487                         );
 488         }
 489 }
 490
 491 list<ReferencedReelAsset>
 492 Player::get_reel_assets ()
 493 {
 494         /* Does not require a lock on _mutex as it's only called from DCPEncoder */
 495
 496         list<ReferencedReelAsset> a;
 497
 498         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 499                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 500                 if (!j) {
 501                         continue;
 502                 }
 503
 504                 scoped_ptr<DCPDecoder> decoder;
 505                 try {
 506                         decoder.reset (new DCPDecoder (_film, j, false));
 507                 } catch (...) {
 508                         return a;
 509                 }
 510
 511                 DCPOMATIC_ASSERT (j->video_frame_rate ());
 512                 double const cfr = j->video_frame_rate().get();
 513                 Frame const trim_start = j->trim_start().frames_round (cfr);
 514                 Frame const trim_end = j->trim_end().frames_round (cfr);
 515                 int const ffr = _film->video_frame_rate ();
 516
 517                 /* position in the asset from the start */
 518                 int64_t offset_from_start = 0;
 519                 /* position in the asset from the end */
 520                 int64_t offset_from_end = 0;
 521                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 522                         /* Assume that main picture duration is the length of the reel */
 523                         offset_from_end += k->main_picture()->actual_duration();
 524                 }
 525
 526                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 527
 528                         /* Assume that main picture duration is the length of the reel */
 529                         int64_t const reel_duration = k->main_picture()->actual_duration();
 530
 531                         /* See doc/design/trim_reels.svg */
 532                         Frame const reel_trim_start = min(reel_duration, max(int64_t(0), trim_start - offset_from_start));
 533                         Frame const reel_trim_end =   min(reel_duration, max(int64_t(0), reel_duration - (offset_from_end - trim_end)));
 534
 535                         DCPTime const from = i->position() + DCPTime::from_frames (offset_from_start, _film->video_frame_rate());
 536                         if (j->reference_video ()) {
 537                                 maybe_add_asset (a, k->main_picture(), reel_trim_start, reel_trim_end, from, ffr);
 538                         }
 539
 540                         if (j->reference_audio ()) {
 541                                 maybe_add_asset (a, k->main_sound(), reel_trim_start, reel_trim_end, from, ffr);
 542                         }
 543
 544                         if (j->reference_text (TEXT_OPEN_SUBTITLE)) {
 545                                 maybe_add_asset (a, k->main_subtitle(), reel_trim_start, reel_trim_end, from, ffr);
 546                         }
 547
 548                         if (j->reference_text (TEXT_CLOSED_CAPTION)) {
 549                                 BOOST_FOREACH (shared_ptr<dcp::ReelClosedCaptionAsset> l, k->closed_captions()) {
 550                                         maybe_add_asset (a, l, reel_trim_start, reel_trim_end, from, ffr);
 551                                 }
 552                         }
 553
 554                         offset_from_start += reel_duration;
 555                         offset_from_end -= reel_duration;
 556                 }
 557         }
 558
 559         return a;
 560 }
 561
 562 bool
 563 Player::pass ()
 564 {
 565         boost::mutex::scoped_lock lm (_mutex);
 566
 567         if (_suspended) {
 568                 /* We can't pass in this state */
 569                 return false;
 570         }
 571
 572         if (_playlist->length(_film) == DCPTime()) {
 573                 /* Special case of an empty Film; just give one black frame */
 574                 emit_video (black_player_video_frame(EYES_BOTH), DCPTime());
 575                 return true;
 576         }
 577
 578         /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
 579
 580         shared_ptr<Piece> earliest_content;
 581         optional<DCPTime> earliest_time;
 582
 583         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 584                 if (i->done) {
 585                         continue;
 586                 }
 587
 588                 DCPTime const t = content_time_to_dcp (i, max(i->decoder->position(), i->content->trim_start()));
 589                 if (t > i->content->end(_film)) {
 590                         i->done = true;
 591                 } else {
 592
 593                         /* Given two choices at the same time, pick the one with texts so we see it before
 594                            the video.
 595                         */
 596                         if (!earliest_time || t < *earliest_time || (t == *earliest_time && !i->decoder->text.empty())) {
 597                                 earliest_time = t;
 598                                 earliest_content = i;
 599                         }
 600                 }
 601         }
 602
 603         bool done = false;
 604
 605         enum {
 606                 NONE,
 607                 CONTENT,
 608                 BLACK,
 609                 SILENT
 610         } which = NONE;
 611
 612         if (earliest_content) {
 613                 which = CONTENT;
 614         }
 615
 616         if (!_black.done() && (!earliest_time || _black.position() < *earliest_time)) {
 617                 earliest_time = _black.position ();
 618                 which = BLACK;
 619         }
 620
 621         if (!_silent.done() && (!earliest_time || _silent.position() < *earliest_time)) {
 622                 earliest_time = _silent.position ();
 623                 which = SILENT;
 624         }
 625
 626         switch (which) {
 627         case CONTENT:
 628         {
 629                 earliest_content->done = earliest_content->decoder->pass ();
 630                 shared_ptr<DCPContent> dcp = dynamic_pointer_cast<DCPContent>(earliest_content->content);
 631                 if (dcp && !_play_referenced && dcp->reference_audio()) {
 632                         /* We are skipping some referenced DCP audio content, so we need to update _last_audio_time
 633                            to `hide' the fact that no audio was emitted during the referenced DCP (though
 634                            we need to behave as though it was).
 635                         */
 636                         _last_audio_time = dcp->end (_film);
 637                 }
 638                 break;
 639         }
 640         case BLACK:
 641                 emit_video (black_player_video_frame(EYES_BOTH), _black.position());
 642                 _black.set_position (_black.position() + one_video_frame());
 643                 break;
 644         case SILENT:
 645         {
 646                 DCPTimePeriod period (_silent.period_at_position());
 647                 if (_last_audio_time) {
 648                         /* Sometimes the thing that happened last finishes fractionally before
 649                            or after this silence.  Bodge the start time of the silence to fix it.
 650                         */
 651                         DCPOMATIC_ASSERT (labs(period.from.get() - _last_audio_time->get()) < 2);
 652                         period.from = *_last_audio_time;
 653                 }
 654                 if (period.duration() > one_video_frame()) {
 655                         period.to = period.from + one_video_frame();
 656                 }
 657                 fill_audio (period);
 658                 _silent.set_position (period.to);
 659                 break;
 660         }
 661         case NONE:
 662                 done = true;
 663                 break;
 664         }
 665
 666         /* Emit any audio that is ready */
 667
 668         /* Work out the time before which the audio is definitely all here.  This is the earliest last_push_end of one
 669            of our streams, or the position of the _silent.
 670         */
 671         DCPTime pull_to = _film->length ();
 672         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 673                 if (!i->second.piece->done && i->second.last_push_end < pull_to) {
 674                         pull_to = i->second.last_push_end;
 675                 }
 676         }
 677         if (!_silent.done() && _silent.position() < pull_to) {
 678                 pull_to = _silent.position();
 679         }
 680
 681         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 682         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 683                 if (_last_audio_time && i->second < *_last_audio_time) {
 684                         /* This new data comes before the last we emitted (or the last seek); discard it */
 685                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 686                         if (!cut.first) {
 687                                 continue;
 688                         }
 689                         *i = cut;
 690                 } else if (_last_audio_time && i->second > *_last_audio_time) {
 691                         /* There's a gap between this data and the last we emitted; fill with silence */
 692                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 693                 }
 694
 695                 emit_audio (i->first, i->second);
 696         }
 697
 698         if (done) {
 699                 _shuffler->flush ();
 700                 for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 701                         do_emit_video(i->first, i->second);
 702                 }
 703         }
 704
 705         return done;
 706 }
 707
 708 /** @return Open subtitles for the frame at the given time, converted to images */
 709 optional<PositionImage>
 710 Player::open_subtitles_for_frame (DCPTime time) const
 711 {
 712         list<PositionImage> captions;
 713         int const vfr = _film->video_frame_rate();
 714
 715         BOOST_FOREACH (
 716                 PlayerText j,
 717                 _active_texts[TEXT_OPEN_SUBTITLE].get_burnt(DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), _always_burn_open_subtitles)
 718                 ) {
 719
 720                 /* Bitmap subtitles */
 721                 BOOST_FOREACH (BitmapText i, j.bitmap) {
 722                         if (!i.image) {
 723                                 continue;
 724                         }
 725
 726                         /* i.image will already have been scaled to fit _video_container_size */
 727                         dcp::Size scaled_size (i.rectangle.width * _video_container_size.width, i.rectangle.height * _video_container_size.height);
 728
 729                         captions.push_back (
 730                                 PositionImage (
 731                                         i.image,
 732                                         Position<int> (
 733                                                 lrint (_video_container_size.width * i.rectangle.x),
 734                                                 lrint (_video_container_size.height * i.rectangle.y)
 735                                                 )
 736                                         )
 737                                 );
 738                 }
 739
 740                 /* String subtitles (rendered to an image) */
 741                 if (!j.string.empty ()) {
 742                         list<PositionImage> s = render_text (j.string, j.fonts, _video_container_size, time, vfr);
 743                         copy (s.begin(), s.end(), back_inserter (captions));
 744                 }
 745         }
 746
 747         if (captions.empty ()) {
 748                 return optional<PositionImage> ();
 749         }
 750
 751         return merge (captions);
 752 }
 753
 754 void
 755 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 756 {
 757         shared_ptr<Piece> piece = wp.lock ();
 758         if (!piece) {
 759                 return;
 760         }
 761
 762         FrameRateChange frc (_film, piece->content);
 763         if (frc.skip && (video.frame % 2) == 1) {
 764                 return;
 765         }
 766
 767         /* Time of the first frame we will emit */
 768         DCPTime const time = content_video_to_dcp (piece, video.frame);
 769
 770         /* Discard if it's before the content's period or the last accurate seek.  We can't discard
 771            if it's after the content's period here as in that case we still need to fill any gap between
 772            `now' and the end of the content's period.
 773         */
 774         if (time < piece->content->position() || (_last_video_time && time < *_last_video_time)) {
 775                 return;
 776         }
 777
 778         /* Fill gaps that we discover now that we have some video which needs to be emitted.
 779            This is where we need to fill to.
 780         */
 781         DCPTime fill_to = min (time, piece->content->end(_film));
 782
 783         if (_last_video_time) {
 784                 DCPTime fill_from = max (*_last_video_time, piece->content->position());
 785
 786                 /* Fill if we have more than half a frame to do */
 787                 if ((fill_to - fill_from) > one_video_frame() / 2) {
 788                         LastVideoMap::const_iterator last = _last_video.find (wp);
 789                         if (_film->three_d()) {
 790                                 Eyes fill_to_eyes = video.eyes;
 791                                 if (fill_to_eyes == EYES_BOTH) {
 792                                         fill_to_eyes = EYES_LEFT;
 793                                 }
 794                                 if (fill_to == piece->content->end(_film)) {
 795                                         /* Don't fill after the end of the content */
 796                                         fill_to_eyes = EYES_LEFT;
 797                                 }
 798                                 DCPTime j = fill_from;
 799                                 Eyes eyes = _last_video_eyes.get_value_or(EYES_LEFT);
 800                                 if (eyes == EYES_BOTH) {
 801                                         eyes = EYES_LEFT;
 802                                 }
 803                                 while (j < fill_to || eyes != fill_to_eyes) {
 804                                         if (last != _last_video.end()) {
 805                                                 shared_ptr<PlayerVideo> copy = last->second->shallow_copy();
 806                                                 copy->set_eyes (eyes);
 807                                                 emit_video (copy, j);
 808                                         } else {
 809                                                 emit_video (black_player_video_frame(eyes), j);
 810                                         }
 811                                         if (eyes == EYES_RIGHT) {
 812                                                 j += one_video_frame();
 813                                         }
 814                                         eyes = increment_eyes (eyes);
 815                                 }
 816                         } else {
 817                                 for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
 818                                         if (last != _last_video.end()) {
 819                                                 emit_video (last->second, j);
 820                                         } else {
 821                                                 emit_video (black_player_video_frame(EYES_BOTH), j);
 822                                         }
 823                                 }
 824                         }
 825                 }
 826         }
 827
 828         _last_video[wp].reset (
 829                 new PlayerVideo (
 830                         video.image,
 831                         piece->content->video->crop (),
 832                         piece->content->video->fade (_film, video.frame),
 833                         piece->content->video->scale().size (
 834                                 piece->content->video, _video_container_size, _film->frame_size ()
 835                                 ),
 836                         _video_container_size,
 837                         video.eyes,
 838                         video.part,
 839                         piece->content->video->colour_conversion(),
 840                         piece->content->video->range(),
 841                         piece->content,
 842                         video.frame
 843                         )
 844                 );
 845
 846         DCPTime t = time;
 847         for (int i = 0; i < frc.repeat; ++i) {
 848                 if (t < piece->content->end(_film)) {
 849                         emit_video (_last_video[wp], t);
 850                 }
 851                 t += one_video_frame ();
 852         }
 853 }
 854
 855 void
 856 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 857 {
 858         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 859
 860         shared_ptr<Piece> piece = wp.lock ();
 861         if (!piece) {
 862                 return;
 863         }
 864
 865         shared_ptr<AudioContent> content = piece->content->audio;
 866         DCPOMATIC_ASSERT (content);
 867
 868         int const rfr = content->resampled_frame_rate (_film);
 869
 870         /* Compute time in the DCP */
 871         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame);
 872         /* And the end of this block in the DCP */
 873         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), rfr);
 874
 875         /* Remove anything that comes before the start or after the end of the content */
 876         if (time < piece->content->position()) {
 877                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 878                 if (!cut.first) {
 879                         /* This audio is entirely discarded */
 880                         return;
 881                 }
 882                 content_audio.audio = cut.first;
 883                 time = cut.second;
 884         } else if (time > piece->content->end(_film)) {
 885                 /* Discard it all */
 886                 return;
 887         } else if (end > piece->content->end(_film)) {
 888                 Frame const remaining_frames = DCPTime(piece->content->end(_film) - time).frames_round(rfr);
 889                 if (remaining_frames == 0) {
 890                         return;
 891                 }
 892                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 893                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 894                 content_audio.audio = cut;
 895         }
 896
 897         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 898
 899         /* Gain */
 900
 901         if (content->gain() != 0) {
 902                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 903                 gain->apply_gain (content->gain ());
 904                 content_audio.audio = gain;
 905         }
 906
 907         /* Remap */
 908
 909         content_audio.audio = remap (content_audio.audio, _film->audio_channels(), stream->mapping());
 910
 911         /* Process */
 912
 913         if (_audio_processor) {
 914                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 915         }
 916
 917         /* Push */
 918
 919         _audio_merger.push (content_audio.audio, time);
 920         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 921         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 922 }
 923
 924 void
 925 Player::bitmap_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentBitmapText subtitle)
 926 {
 927         shared_ptr<Piece> piece = wp.lock ();
 928         shared_ptr<const TextContent> text = wc.lock ();
 929         if (!piece || !text) {
 930                 return;
 931         }
 932
 933         /* Apply content's subtitle offsets */
 934         subtitle.sub.rectangle.x += text->x_offset ();
 935         subtitle.sub.rectangle.y += text->y_offset ();
 936
 937         /* Apply a corrective translation to keep the subtitle centred after the scale that is coming up */
 938         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * ((text->x_scale() - 1) / 2);
 939         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * ((text->y_scale() - 1) / 2);
 940
 941         /* Apply content's subtitle scale */
 942         subtitle.sub.rectangle.width *= text->x_scale ();
 943         subtitle.sub.rectangle.height *= text->y_scale ();
 944
 945         PlayerText ps;
 946         shared_ptr<Image> image = subtitle.sub.image;
 947         /* We will scale the subtitle up to fit _video_container_size */
 948         dcp::Size scaled_size (subtitle.sub.rectangle.width * _video_container_size.width, subtitle.sub.rectangle.height * _video_container_size.height);
 949         ps.bitmap.push_back (BitmapText(image->scale(scaled_size, dcp::YUV_TO_RGB_REC601, image->pixel_format(), true, _fast), subtitle.sub.rectangle));
 950         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 951
 952         _active_texts[text->type()].add_from (wc, ps, from);
 953 }
 954
 955 void
 956 Player::plain_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentStringText subtitle)
 957 {
 958         shared_ptr<Piece> piece = wp.lock ();
 959         shared_ptr<const TextContent> text = wc.lock ();
 960         if (!piece || !text) {
 961                 return;
 962         }
 963
 964         PlayerText ps;
 965         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 966
 967         if (from > piece->content->end(_film)) {
 968                 return;
 969         }
 970
 971         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 972                 s.set_h_position (s.h_position() + text->x_offset ());
 973                 s.set_v_position (s.v_position() + text->y_offset ());
 974                 float const xs = text->x_scale();
 975                 float const ys = text->y_scale();
 976                 float size = s.size();
 977
 978                 /* Adjust size to express the common part of the scaling;
 979                    e.g. if xs = ys = 0.5 we scale size by 2.
 980                 */
 981                 if (xs > 1e-5 && ys > 1e-5) {
 982                         size *= 1 / min (1 / xs, 1 / ys);
 983                 }
 984                 s.set_size (size);
 985
 986                 /* Then express aspect ratio changes */
 987                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 988                         s.set_aspect_adjust (xs / ys);
 989                 }
 990
 991                 s.set_in (dcp::Time(from.seconds(), 1000));
 992                 ps.string.push_back (StringText (s, text->outline_width()));
 993                 ps.add_fonts (text->fonts ());
 994         }
 995
 996         _active_texts[text->type()].add_from (wc, ps, from);
 997 }
 998
 999 void
1000 Player::subtitle_stop (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentTime to)
1001 {
1002         shared_ptr<const TextContent> text = wc.lock ();
1003         if (!text) {
1004                 return;
1005         }
1006
1007         if (!_active_texts[text->type()].have(wc)) {
1008                 return;
1009         }
1010
1011         shared_ptr<Piece> piece = wp.lock ();
1012         if (!piece) {
1013                 return;
1014         }
1015
1016         DCPTime const dcp_to = content_time_to_dcp (piece, to);
1017
1018         if (dcp_to > piece->content->end(_film)) {
1019                 return;
1020         }
1021
1022         pair<PlayerText, DCPTime> from = _active_texts[text->type()].add_to (wc, dcp_to);
1023
1024         bool const always = (text->type() == TEXT_OPEN_SUBTITLE && _always_burn_open_subtitles);
1025         if (text->use() && !always && !text->burn()) {
1026                 Text (from.first, text->type(), text->dcp_track().get_value_or(DCPTextTrack()), DCPTimePeriod (from.second, dcp_to));
1027         }
1028 }
1029
1030 void
1031 Player::seek (DCPTime time, bool accurate)
1032 {
1033         boost::mutex::scoped_lock lm (_mutex);
1034
1035         if (_suspended) {
1036                 /* We can't seek in this state */
1037                 return;
1038         }
1039
1040         if (_shuffler) {
1041                 _shuffler->clear ();
1042         }
1043
1044         _delay.clear ();
1045
1046         if (_audio_processor) {
1047                 _audio_processor->flush ();
1048         }
1049
1050         _audio_merger.clear ();
1051         for (int i = 0; i < TEXT_COUNT; ++i) {
1052                 _active_texts[i].clear ();
1053         }
1054
1055         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1056                 if (time < i->content->position()) {
1057                         /* Before; seek to the start of the content.  Even if this request is for an inaccurate seek
1058                            we must seek this (following) content accurately, otherwise when we come to the end of the current
1059                            content we may not start right at the beginning of the next, causing a gap (if the next content has
1060                            been trimmed to a point between keyframes, or something).
1061                         */
1062                         i->decoder->seek (dcp_to_content_time (i, i->content->position()), true);
1063                         i->done = false;
1064                 } else if (i->content->position() <= time && time < i->content->end(_film)) {
1065                         /* During; seek to position */
1066                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
1067                         i->done = false;
1068                 } else {
1069                         /* After; this piece is done */
1070                         i->done = true;
1071                 }
1072         }
1073
1074         if (accurate) {
1075                 _last_video_time = time;
1076                 _last_video_eyes = EYES_LEFT;
1077                 _last_audio_time = time;
1078         } else {
1079                 _last_video_time = optional<DCPTime>();
1080                 _last_video_eyes = optional<Eyes>();
1081                 _last_audio_time = optional<DCPTime>();
1082         }
1083
1084         _black.set_position (time);
1085         _silent.set_position (time);
1086
1087         _last_video.clear ();
1088 }
1089
1090 void
1091 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1092 {
1093         /* We need a delay to give a little wiggle room to ensure that relevent subtitles arrive at the
1094            player before the video that requires them.
1095         */
1096         _delay.push_back (make_pair (pv, time));
1097
1098         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1099                 _last_video_time = time + one_video_frame();
1100         }
1101         _last_video_eyes = increment_eyes (pv->eyes());
1102
1103         if (_delay.size() < 3) {
1104                 return;
1105         }
1106
1107         pair<shared_ptr<PlayerVideo>, DCPTime> to_do = _delay.front();
1108         _delay.pop_front();
1109         do_emit_video (to_do.first, to_do.second);
1110 }
1111
1112 void
1113 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1114 {
1115         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1116                 for (int i = 0; i < TEXT_COUNT; ++i) {
1117                         _active_texts[i].clear_before (time);
1118                 }
1119         }
1120
1121         optional<PositionImage> subtitles = open_subtitles_for_frame (time);
1122         if (subtitles) {
1123                 pv->set_text (subtitles.get ());
1124         }
1125
1126         Video (pv, time);
1127 }
1128
1129 void
1130 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1131 {
1132         /* Log if the assert below is about to fail */
1133         if (_last_audio_time && labs(time.get() - _last_audio_time->get()) > 1) {
1134                 _film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_last_audio_time)), LogEntry::TYPE_WARNING);
1135         }
1136
1137         /* This audio must follow on from the previous, allowing for half a sample (at 48kHz) leeway */
1138         DCPOMATIC_ASSERT (!_last_audio_time || labs(time.get() - _last_audio_time->get()) < 2);
1139         Audio (data, time, _film->audio_frame_rate());
1140         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate());
1141 }
1142
1143 void
1144 Player::fill_audio (DCPTimePeriod period)
1145 {
1146         if (period.from == period.to) {
1147                 return;
1148         }
1149
1150         DCPOMATIC_ASSERT (period.from < period.to);
1151
1152         DCPTime t = period.from;
1153         while (t < period.to) {
1154                 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1155                 Frame const samples = block.frames_round(_film->audio_frame_rate());
1156                 if (samples) {
1157                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1158                         silence->make_silent ();
1159                         emit_audio (silence, t);
1160                 }
1161                 t += block;
1162         }
1163 }
1164
1165 DCPTime
1166 Player::one_video_frame () const
1167 {
1168         return DCPTime::from_frames (1, _film->video_frame_rate ());
1169 }
1170
1171 pair<shared_ptr<AudioBuffers>, DCPTime>
1172 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1173 {
1174         DCPTime const discard_time = discard_to - time;
1175         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1176         Frame remaining_frames = audio->frames() - discard_frames;
1177         if (remaining_frames <= 0) {
1178                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1179         }
1180         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1181         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1182         return make_pair(cut, time + discard_time);
1183 }
1184
1185 void
1186 Player::set_dcp_decode_reduction (optional<int> reduction)
1187 {
1188         Change (CHANGE_TYPE_PENDING, PlayerProperty::DCP_DECODE_REDUCTION, false);
1189
1190         {
1191                 boost::mutex::scoped_lock lm (_mutex);
1192
1193                 if (reduction == _dcp_decode_reduction) {
1194                         lm.unlock ();
1195                         Change (CHANGE_TYPE_CANCELLED, PlayerProperty::DCP_DECODE_REDUCTION, false);
1196                         return;
1197                 }
1198
1199                 _dcp_decode_reduction = reduction;
1200                 setup_pieces_unlocked ();
1201         }
1202
1203         Change (CHANGE_TYPE_DONE, PlayerProperty::DCP_DECODE_REDUCTION, false);
1204 }
1205
1206 optional<DCPTime>
1207 Player::content_time_to_dcp (shared_ptr<Content> content, ContentTime t)
1208 {
1209         boost::mutex::scoped_lock lm (_mutex);
1210
1211         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1212                 if (i->content == content) {
1213                         return content_time_to_dcp (i, t);
1214                 }
1215         }
1216
1217         /* We couldn't find this content; perhaps things are being changed over */
1218         return optional<DCPTime>();
1219 }