src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2020 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "atmos_decoder.h"
  22 #include "player.h"
  23 #include "film.h"
  24 #include "audio_buffers.h"
  25 #include "content_audio.h"
  26 #include "dcp_content.h"
  27 #include "dcpomatic_log.h"
  28 #include "job.h"
  29 #include "image.h"
  30 #include "raw_image_proxy.h"
  31 #include "ratio.h"
  32 #include "log.h"
  33 #include "render_text.h"
  34 #include "config.h"
  35 #include "content_video.h"
  36 #include "player_video.h"
  37 #include "frame_rate_change.h"
  38 #include "audio_processor.h"
  39 #include "playlist.h"
  40 #include "referenced_reel_asset.h"
  41 #include "decoder_factory.h"
  42 #include "decoder.h"
  43 #include "video_decoder.h"
  44 #include "audio_decoder.h"
  45 #include "text_content.h"
  46 #include "text_decoder.h"
  47 #include "ffmpeg_content.h"
  48 #include "audio_content.h"
  49 #include "dcp_decoder.h"
  50 #include "image_decoder.h"
  51 #include "compose.hpp"
  52 #include "shuffler.h"
  53 #include "timer.h"
  54 #include <dcp/reel.h>
  55 #include <dcp/reel_sound_asset.h>
  56 #include <dcp/reel_subtitle_asset.h>
  57 #include <dcp/reel_picture_asset.h>
  58 #include <dcp/reel_closed_caption_asset.h>
  59 #include <boost/foreach.hpp>
  60 #include <stdint.h>
  61 #include <algorithm>
  62 #include <iostream>
  63
  64 #include "i18n.h"
  65
  66 using std::list;
  67 using std::cout;
  68 using std::min;
  69 using std::max;
  70 using std::min;
  71 using std::vector;
  72 using std::pair;
  73 using std::map;
  74 using std::make_pair;
  75 using std::copy;
  76 using boost::shared_ptr;
  77 using boost::weak_ptr;
  78 using boost::dynamic_pointer_cast;
  79 using boost::optional;
  80 using boost::scoped_ptr;
  81 #if BOOST_VERSION >= 106100
  82 using namespace boost::placeholders;
  83 #endif
  84 using namespace dcpomatic;
  85
  86 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
  87 int const PlayerProperty::PLAYLIST = 701;
  88 int const PlayerProperty::FILM_CONTAINER = 702;
  89 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
  90 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
  91 int const PlayerProperty::PLAYBACK_LENGTH = 705;
  92
  93 Player::Player (shared_ptr<const Film> film)
  94         : _film (film)
  95         , _suspended (0)
  96         , _ignore_video (false)
  97         , _ignore_audio (false)
  98         , _ignore_text (false)
  99         , _always_burn_open_subtitles (false)
 100         , _fast (false)
 101         , _tolerant (film->tolerant())
 102         , _play_referenced (false)
 103         , _audio_merger (_film->audio_frame_rate())
 104         , _shuffler (0)
 105 {
 106         construct ();
 107 }
 108
 109 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist_)
 110         : _film (film)
 111         , _playlist (playlist_)
 112         , _suspended (0)
 113         , _ignore_video (false)
 114         , _ignore_audio (false)
 115         , _ignore_text (false)
 116         , _always_burn_open_subtitles (false)
 117         , _fast (false)
 118         , _tolerant (film->tolerant())
 119         , _play_referenced (false)
 120         , _audio_merger (_film->audio_frame_rate())
 121         , _shuffler (0)
 122 {
 123         construct ();
 124 }
 125
 126 void
 127 Player::construct ()
 128 {
 129         _film_changed_connection = _film->Change.connect (bind (&Player::film_change, this, _1, _2));
 130         /* The butler must hear about this first, so since we are proxying this through to the butler we must
 131            be first.
 132         */
 133         _playlist_change_connection = playlist()->Change.connect (bind (&Player::playlist_change, this, _1), boost::signals2::at_front);
 134         _playlist_content_change_connection = playlist()->ContentChange.connect (bind(&Player::playlist_content_change, this, _1, _3, _4));
 135         set_video_container_size (_film->frame_size ());
 136
 137         film_change (CHANGE_TYPE_DONE, Film::AUDIO_PROCESSOR);
 138
 139         setup_pieces ();
 140         seek (DCPTime (), true);
 141 }
 142
 143 Player::~Player ()
 144 {
 145         delete _shuffler;
 146 }
 147
 148 void
 149 Player::setup_pieces ()
 150 {
 151         boost::mutex::scoped_lock lm (_mutex);
 152         setup_pieces_unlocked ();
 153 }
 154
 155
 156 bool
 157 have_video (shared_ptr<const Content> content)
 158 {
 159         return static_cast<bool>(content->video) && content->video->use();
 160 }
 161
 162 bool
 163 have_audio (shared_ptr<const Content> content)
 164 {
 165         return static_cast<bool>(content->audio);
 166 }
 167
 168 void
 169 Player::setup_pieces_unlocked ()
 170 {
 171         _playback_length = _playlist ? _playlist->length(_film) : _film->length();
 172
 173         list<shared_ptr<Piece> > old_pieces = _pieces;
 174         _pieces.clear ();
 175
 176         delete _shuffler;
 177         _shuffler = new Shuffler();
 178         _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
 179
 180         BOOST_FOREACH (shared_ptr<Content> i, playlist()->content()) {
 181
 182                 if (!i->paths_valid ()) {
 183                         continue;
 184                 }
 185
 186                 if (_ignore_video && _ignore_audio && i->text.empty()) {
 187                         /* We're only interested in text and this content has none */
 188                         continue;
 189                 }
 190
 191                 shared_ptr<Decoder> old_decoder;
 192                 /* XXX: needs to check vector of Content and use the old decoders, but
 193                  * this will all be different as we have to coalesce content before
 194                  * this happens.
 195                 BOOST_FOREACH (shared_ptr<Piece> j, old_pieces) {
 196                         if (j->content == i) {
 197                                 old_decoder = j->decoder;
 198                                 break;
 199                         }
 200                 }
 201                 */
 202
 203                 shared_ptr<Decoder> decoder = decoder_factory (_film, i, _fast, _tolerant, old_decoder);
 204                 DCPOMATIC_ASSERT (decoder);
 205
 206                 FrameRateChange frc (_film, i);
 207
 208                 if (decoder->video && _ignore_video) {
 209                         decoder->video->set_ignore (true);
 210                 }
 211
 212                 if (decoder->audio && _ignore_audio) {
 213                         decoder->audio->set_ignore (true);
 214                 }
 215
 216                 if (_ignore_text) {
 217                         BOOST_FOREACH (shared_ptr<TextDecoder> i, decoder->text) {
 218                                 i->set_ignore (true);
 219                         }
 220                 }
 221
 222                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 223                 if (dcp) {
 224                         dcp->set_decode_referenced (_play_referenced);
 225                         if (_play_referenced) {
 226                                 dcp->set_forced_reduction (_dcp_decode_reduction);
 227                         }
 228                 }
 229
 230                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 231                 _pieces.push_back (piece);
 232
 233                 if (decoder->video) {
 234                         if (i->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT || i->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) {
 235                                 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
 236                                 decoder->video->Data.connect (bind (&Shuffler::video, _shuffler, weak_ptr<Piece>(piece), _1));
 237                         } else {
 238                                 decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece>(piece), _1));
 239                         }
 240                 }
 241
 242                 if (decoder->audio) {
 243                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 244                 }
 245
 246                 list<shared_ptr<TextDecoder> >::const_iterator j = decoder->text.begin();
 247
 248                 while (j != decoder->text.end()) {
 249                         (*j)->BitmapStart.connect (
 250                                 bind(&Player::bitmap_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 251                                 );
 252                         (*j)->PlainStart.connect (
 253                                 bind(&Player::plain_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 254                                 );
 255                         (*j)->Stop.connect (
 256                                 bind(&Player::subtitle_stop, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 257                                 );
 258
 259                         ++j;
 260                 }
 261
 262                 if (decoder->atmos) {
 263                         decoder->atmos->Data.connect (bind(&Player::atmos, this, weak_ptr<Piece>(piece), _1));
 264                 }
 265         }
 266
 267         _black = Empty (_film, playlist(), bind(&have_video, _1), _playback_length);
 268         _silent = Empty (_film, playlist(), bind(&have_audio, _1), _playback_length);
 269
 270         _last_video_time = DCPTime ();
 271         _last_video_eyes = EYES_BOTH;
 272         _last_audio_time = DCPTime ();
 273 }
 274
 275 void
 276 Player::playlist_content_change (ChangeType type, int property, bool frequent)
 277 {
 278         if (property == VideoContentProperty::CROP) {
 279                 if (type == CHANGE_TYPE_DONE) {
 280                         dcp::Size const vcs = video_container_size();
 281                         boost::mutex::scoped_lock lm (_mutex);
 282                         for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 283                                 i->first->reset_metadata (_film, vcs);
 284                         }
 285                 }
 286         } else {
 287                 if (type == CHANGE_TYPE_PENDING) {
 288                         /* The player content is probably about to change, so we can't carry on
 289                            until that has happened and we've rebuilt our pieces.  Stop pass()
 290                            and seek() from working until then.
 291                         */
 292                         ++_suspended;
 293                 } else if (type == CHANGE_TYPE_DONE) {
 294                         /* A change in our content has gone through.  Re-build our pieces. */
 295                         setup_pieces ();
 296                         --_suspended;
 297                 } else if (type == CHANGE_TYPE_CANCELLED) {
 298                         --_suspended;
 299                 }
 300         }
 301
 302         Change (type, property, frequent);
 303 }
 304
 305 void
 306 Player::set_video_container_size (dcp::Size s)
 307 {
 308         Change (CHANGE_TYPE_PENDING, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 309
 310         {
 311                 boost::mutex::scoped_lock lm (_mutex);
 312
 313                 if (s == _video_container_size) {
 314                         lm.unlock ();
 315                         Change (CHANGE_TYPE_CANCELLED, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 316                         return;
 317                 }
 318
 319                 _video_container_size = s;
 320
 321                 _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 322                 _black_image->make_black ();
 323         }
 324
 325         Change (CHANGE_TYPE_DONE, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 326 }
 327
 328 void
 329 Player::playlist_change (ChangeType type)
 330 {
 331         if (type == CHANGE_TYPE_DONE) {
 332                 setup_pieces ();
 333         }
 334         Change (type, PlayerProperty::PLAYLIST, false);
 335 }
 336
 337 void
 338 Player::film_change (ChangeType type, Film::Property p)
 339 {
 340         /* Here we should notice Film properties that affect our output, and
 341            alert listeners that our output now would be different to how it was
 342            last time we were run.
 343         */
 344
 345         if (p == Film::CONTAINER) {
 346                 Change (type, PlayerProperty::FILM_CONTAINER, false);
 347         } else if (p == Film::VIDEO_FRAME_RATE) {
 348                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 349                    so we need new pieces here.
 350                 */
 351                 if (type == CHANGE_TYPE_DONE) {
 352                         setup_pieces ();
 353                 }
 354                 Change (type, PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
 355         } else if (p == Film::AUDIO_PROCESSOR) {
 356                 if (type == CHANGE_TYPE_DONE && _film->audio_processor ()) {
 357                         boost::mutex::scoped_lock lm (_mutex);
 358                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 359                 }
 360         } else if (p == Film::AUDIO_CHANNELS) {
 361                 if (type == CHANGE_TYPE_DONE) {
 362                         boost::mutex::scoped_lock lm (_mutex);
 363                         _audio_merger.clear ();
 364                 }
 365         }
 366 }
 367
 368 shared_ptr<PlayerVideo>
 369 Player::black_player_video_frame (Eyes eyes) const
 370 {
 371         return shared_ptr<PlayerVideo> (
 372                 new PlayerVideo (
 373                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 374                         Crop (),
 375                         optional<double> (),
 376                         _video_container_size,
 377                         _video_container_size,
 378                         eyes,
 379                         PART_WHOLE,
 380                         PresetColourConversion::all().front().conversion,
 381                         VIDEO_RANGE_FULL,
 382                         boost::weak_ptr<Content>(),
 383                         boost::optional<Frame>(),
 384                         false
 385                 )
 386         );
 387 }
 388
 389
 390 list<shared_ptr<Font> >
 391 Player::get_subtitle_fonts ()
 392 {
 393         boost::mutex::scoped_lock lm (_mutex);
 394
 395         list<shared_ptr<Font> > fonts;
 396         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 397                 i->add_fonts (fonts);
 398         }
 399
 400         return fonts;
 401 }
 402
 403 /** Set this player never to produce any video data */
 404 void
 405 Player::set_ignore_video ()
 406 {
 407         boost::mutex::scoped_lock lm (_mutex);
 408         _ignore_video = true;
 409         setup_pieces_unlocked ();
 410 }
 411
 412 void
 413 Player::set_ignore_audio ()
 414 {
 415         boost::mutex::scoped_lock lm (_mutex);
 416         _ignore_audio = true;
 417         setup_pieces_unlocked ();
 418 }
 419
 420 void
 421 Player::set_ignore_text ()
 422 {
 423         boost::mutex::scoped_lock lm (_mutex);
 424         _ignore_text = true;
 425         setup_pieces_unlocked ();
 426 }
 427
 428 /** Set the player to always burn open texts into the image regardless of the content settings */
 429 void
 430 Player::set_always_burn_open_subtitles ()
 431 {
 432         boost::mutex::scoped_lock lm (_mutex);
 433         _always_burn_open_subtitles = true;
 434 }
 435
 436 /** Sets up the player to be faster, possibly at the expense of quality */
 437 void
 438 Player::set_fast ()
 439 {
 440         boost::mutex::scoped_lock lm (_mutex);
 441         _fast = true;
 442         setup_pieces_unlocked ();
 443 }
 444
 445 void
 446 Player::set_play_referenced ()
 447 {
 448         boost::mutex::scoped_lock lm (_mutex);
 449         _play_referenced = true;
 450         setup_pieces_unlocked ();
 451 }
 452
 453 static void
 454 maybe_add_asset (list<ReferencedReelAsset>& a, shared_ptr<dcp::ReelAsset> r, Frame reel_trim_start, Frame reel_trim_end, DCPTime from, int const ffr)
 455 {
 456         DCPOMATIC_ASSERT (r);
 457         r->set_entry_point (r->entry_point().get_value_or(0) + reel_trim_start);
 458         r->set_duration (r->actual_duration() - reel_trim_start - reel_trim_end);
 459         if (r->actual_duration() > 0) {
 460                 a.push_back (
 461                         ReferencedReelAsset(r, DCPTimePeriod(from, from + DCPTime::from_frames(r->actual_duration(), ffr)))
 462                         );
 463         }
 464 }
 465
 466 list<ReferencedReelAsset>
 467 Player::get_reel_assets ()
 468 {
 469         /* Does not require a lock on _mutex as it's only called from DCPEncoder */
 470
 471         list<ReferencedReelAsset> a;
 472
 473         BOOST_FOREACH (shared_ptr<Content> i, playlist()->content()) {
 474                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 475                 if (!j) {
 476                         continue;
 477                 }
 478
 479                 scoped_ptr<DCPDecoder> decoder;
 480                 try {
 481                         decoder.reset (new DCPDecoder (_film, j, false, false, shared_ptr<DCPDecoder>()));
 482                 } catch (...) {
 483                         return a;
 484                 }
 485
 486                 DCPOMATIC_ASSERT (j->video_frame_rate ());
 487                 double const cfr = j->video_frame_rate().get();
 488                 Frame const trim_start = j->trim_start().frames_round (cfr);
 489                 Frame const trim_end = j->trim_end().frames_round (cfr);
 490                 int const ffr = _film->video_frame_rate ();
 491
 492                 /* position in the asset from the start */
 493                 int64_t offset_from_start = 0;
 494                 /* position in the asset from the end */
 495                 int64_t offset_from_end = 0;
 496                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 497                         /* Assume that main picture duration is the length of the reel */
 498                         offset_from_end += k->main_picture()->actual_duration();
 499                 }
 500
 501                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 502
 503                         /* Assume that main picture duration is the length of the reel */
 504                         int64_t const reel_duration = k->main_picture()->actual_duration();
 505
 506                         /* See doc/design/trim_reels.svg */
 507                         Frame const reel_trim_start = min(reel_duration, max(int64_t(0), trim_start - offset_from_start));
 508                         Frame const reel_trim_end =   min(reel_duration, max(int64_t(0), reel_duration - (offset_from_end - trim_end)));
 509
 510                         DCPTime const from = i->position() + DCPTime::from_frames (offset_from_start, _film->video_frame_rate());
 511                         if (j->reference_video ()) {
 512                                 maybe_add_asset (a, k->main_picture(), reel_trim_start, reel_trim_end, from, ffr);
 513                         }
 514
 515                         if (j->reference_audio ()) {
 516                                 maybe_add_asset (a, k->main_sound(), reel_trim_start, reel_trim_end, from, ffr);
 517                         }
 518
 519                         if (j->reference_text (TEXT_OPEN_SUBTITLE)) {
 520                                 maybe_add_asset (a, k->main_subtitle(), reel_trim_start, reel_trim_end, from, ffr);
 521                         }
 522
 523                         if (j->reference_text (TEXT_CLOSED_CAPTION)) {
 524                                 BOOST_FOREACH (shared_ptr<dcp::ReelClosedCaptionAsset> l, k->closed_captions()) {
 525                                         maybe_add_asset (a, l, reel_trim_start, reel_trim_end, from, ffr);
 526                                 }
 527                         }
 528
 529                         offset_from_start += reel_duration;
 530                         offset_from_end -= reel_duration;
 531                 }
 532         }
 533
 534         return a;
 535 }
 536
 537 bool
 538 Player::pass ()
 539 {
 540         boost::mutex::scoped_lock lm (_mutex);
 541
 542         if (_suspended) {
 543                 /* We can't pass in this state */
 544                 LOG_DEBUG_PLAYER_NC ("Player is suspended");
 545                 return false;
 546         }
 547
 548         if (_playback_length == DCPTime()) {
 549                 /* Special; just give one black frame */
 550                 emit_video (black_player_video_frame(EYES_BOTH), DCPTime());
 551                 return true;
 552         }
 553
 554         /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
 555
 556         shared_ptr<Piece> earliest_content;
 557         optional<DCPTime> earliest_time;
 558
 559         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 560                 optional<DCPTime> t = i->position ();
 561                 /* Given two choices at the same time, pick the one with texts so we see it before
 562                    the video.
 563                    */
 564                 if (t) {
 565                         if (!earliest_time || *t < *earliest_time || (*t == *earliest_time && i->has_text())) {
 566                                 earliest_content = i;
 567                                 earliest_time = *t;
 568                         }
 569                 }
 570         }
 571
 572         bool done = false;
 573
 574         enum {
 575                 NONE,
 576                 CONTENT,
 577                 BLACK,
 578                 SILENT
 579         } which = NONE;
 580
 581         if (earliest_content) {
 582                 which = CONTENT;
 583         }
 584
 585         if (!_black.done() && !_ignore_video && (!earliest_time || _black.position() < *earliest_time)) {
 586                 earliest_time = _black.position ();
 587                 which = BLACK;
 588         }
 589
 590         if (!_silent.done() && !_ignore_audio && (!earliest_time || _silent.position() < *earliest_time)) {
 591                 earliest_time = _silent.position ();
 592                 which = SILENT;
 593         }
 594
 595         switch (which) {
 596         case CONTENT:
 597         {
 598                 LOG_DEBUG_PLAYER ("Calling pass() on %1", earliest_content->content->path(0));
 599                 earliest_content->pass ();
 600                 if (!_play_referenced && earliest_content->reference_audio()) {
 601                         /* We are skipping some referenced DCP audio content, so we need to update _last_audio_time
 602                            to `hide' the fact that no audio was emitted during the referenced DCP (though
 603                            we need to behave as though it was).
 604                         */
 605                         _last_audio_time = earliest_content->end (_film);
 606                 }
 607                 break;
 608         }
 609         case BLACK:
 610                 LOG_DEBUG_PLAYER ("Emit black for gap at %1", to_string(_black.position()));
 611                 emit_video (black_player_video_frame(EYES_BOTH), _black.position());
 612                 _black.set_position (_black.position() + one_video_frame());
 613                 break;
 614         case SILENT:
 615         {
 616                 LOG_DEBUG_PLAYER ("Emit silence for gap at %1", to_string(_silent.position()));
 617                 DCPTimePeriod period (_silent.period_at_position());
 618                 if (_last_audio_time) {
 619                         /* Sometimes the thing that happened last finishes fractionally before
 620                            or after this silence.  Bodge the start time of the silence to fix it.
 621                            I think this is nothing to worry about since we will just add or
 622                            remove a little silence at the end of some content.
 623                         */
 624                         int64_t const error = labs(period.from.get() - _last_audio_time->get());
 625                         /* Let's not worry about less than a frame at 24fps */
 626                         int64_t const too_much_error = DCPTime::from_frames(1, 24).get();
 627                         if (error >= too_much_error) {
 628                                 _film->log()->log(String::compose("Silence starting before or after last audio by %1", error), LogEntry::TYPE_ERROR);
 629                         }
 630                         DCPOMATIC_ASSERT (error < too_much_error);
 631                         period.from = *_last_audio_time;
 632                 }
 633                 if (period.duration() > one_video_frame()) {
 634                         period.to = period.from + one_video_frame();
 635                 }
 636                 fill_audio (period);
 637                 _silent.set_position (period.to);
 638                 break;
 639         }
 640         case NONE:
 641                 done = true;
 642                 break;
 643         }
 644
 645         /* Emit any audio that is ready */
 646
 647         /* Work out the time before which the audio is definitely all here.  This is the earliest last_push_end of one
 648            of our streams, or the position of the _silent.
 649         */
 650         DCPTime pull_to = _playback_length;
 651         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 652                 i->update_pull_to (pull_to);
 653         }
 654         if (!_silent.done() && _silent.position() < pull_to) {
 655                 pull_to = _silent.position();
 656         }
 657
 658         LOG_DEBUG_PLAYER("Emitting audio up to %1", to_string(pull_to));
 659         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 660         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 661                 if (_last_audio_time && i->second < *_last_audio_time) {
 662                         /* This new data comes before the last we emitted (or the last seek); discard it */
 663                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 664                         if (!cut.first) {
 665                                 continue;
 666                         }
 667                         *i = cut;
 668                 } else if (_last_audio_time && i->second > *_last_audio_time) {
 669                         /* There's a gap between this data and the last we emitted; fill with silence */
 670                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 671                 }
 672
 673                 emit_audio (i->first, i->second);
 674         }
 675
 676         if (done) {
 677                 _shuffler->flush ();
 678                 for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 679                         do_emit_video(i->first, i->second);
 680                 }
 681         }
 682
 683         return done;
 684 }
 685
 686 /** @return Open subtitles for the frame at the given time, converted to images */
 687 optional<PositionImage>
 688 Player::open_subtitles_for_frame (DCPTime time) const
 689 {
 690         list<PositionImage> captions;
 691         int const vfr = _film->video_frame_rate();
 692
 693         BOOST_FOREACH (
 694                 PlayerText j,
 695                 _active_texts[TEXT_OPEN_SUBTITLE].get_burnt(DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), _always_burn_open_subtitles)
 696                 ) {
 697
 698                 /* Bitmap subtitles */
 699                 BOOST_FOREACH (BitmapText i, j.bitmap) {
 700                         if (!i.image) {
 701                                 continue;
 702                         }
 703
 704                         /* i.image will already have been scaled to fit _video_container_size */
 705                         dcp::Size scaled_size (i.rectangle.width * _video_container_size.width, i.rectangle.height * _video_container_size.height);
 706
 707                         captions.push_back (
 708                                 PositionImage (
 709                                         i.image,
 710                                         Position<int> (
 711                                                 lrint (_video_container_size.width * i.rectangle.x),
 712                                                 lrint (_video_container_size.height * i.rectangle.y)
 713                                                 )
 714                                         )
 715                                 );
 716                 }
 717
 718                 /* String subtitles (rendered to an image) */
 719                 if (!j.string.empty ()) {
 720                         list<PositionImage> s = render_text (j.string, j.fonts, _video_container_size, time, vfr);
 721                         copy (s.begin(), s.end(), back_inserter (captions));
 722                 }
 723         }
 724
 725         if (captions.empty ()) {
 726                 return optional<PositionImage> ();
 727         }
 728
 729         return merge (captions);
 730 }
 731
 732 void
 733 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 734 {
 735         shared_ptr<Piece> piece = wp.lock ();
 736         if (!piece) {
 737                 return;
 738         }
 739
 740         if (!piece->video_use()) {
 741                 return;
 742         }
 743
 744         FrameRateChange const frc = piece->frame_rate_change();
 745         if (frc.skip && (video.frame % 2) == 1) {
 746                 return;
 747         }
 748
 749         /* Time of the first frame we will emit */
 750         DCPTime const time = piece->content_video_to_dcp (video.frame);
 751         LOG_DEBUG_PLAYER("Received video frame %1 at %2", video.frame, to_string(time));
 752
 753         /* Discard if it's before the content's period or the last accurate seek.  We can't discard
 754            if it's after the content's period here as in that case we still need to fill any gap between
 755            `now' and the end of the content's period.
 756         */
 757         if (time < piece->position() || (_last_video_time && time < *_last_video_time)) {
 758                 return;
 759         }
 760
 761         /* Fill gaps that we discover now that we have some video which needs to be emitted.
 762            This is where we need to fill to.
 763         */
 764         DCPTime fill_to = min (time, piece->end(_film));
 765
 766         if (_last_video_time) {
 767                 DCPTime fill_from = max (*_last_video_time, piece->position());
 768
 769                 /* Fill if we have more than half a frame to do */
 770                 if ((fill_to - fill_from) > one_video_frame() / 2) {
 771                         LastVideoMap::const_iterator last = _last_video.find (wp);
 772                         if (_film->three_d()) {
 773                                 Eyes fill_to_eyes = video.eyes;
 774                                 if (fill_to_eyes == EYES_BOTH) {
 775                                         fill_to_eyes = EYES_LEFT;
 776                                 }
 777                                 if (fill_to == piece->end(_film)) {
 778                                         /* Don't fill after the end of the content */
 779                                         fill_to_eyes = EYES_LEFT;
 780                                 }
 781                                 DCPTime j = fill_from;
 782                                 Eyes eyes = _last_video_eyes.get_value_or(EYES_LEFT);
 783                                 if (eyes == EYES_BOTH) {
 784                                         eyes = EYES_LEFT;
 785                                 }
 786                                 while (j < fill_to || eyes != fill_to_eyes) {
 787                                         if (last != _last_video.end()) {
 788                                                 LOG_DEBUG_PLAYER("Fill using last video at %1 in 3D mode", to_string(j));
 789                                                 shared_ptr<PlayerVideo> copy = last->second->shallow_copy();
 790                                                 copy->set_eyes (eyes);
 791                                                 emit_video (copy, j);
 792                                         } else {
 793                                                 LOG_DEBUG_PLAYER("Fill using black at %1 in 3D mode", to_string(j));
 794                                                 emit_video (black_player_video_frame(eyes), j);
 795                                         }
 796                                         if (eyes == EYES_RIGHT) {
 797                                                 j += one_video_frame();
 798                                         }
 799                                         eyes = increment_eyes (eyes);
 800                                 }
 801                         } else {
 802                                 for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
 803                                         if (last != _last_video.end()) {
 804                                                 emit_video (last->second, j);
 805                                         } else {
 806                                                 emit_video (black_player_video_frame(EYES_BOTH), j);
 807                                         }
 808                                 }
 809                         }
 810                 }
 811         }
 812
 813         _last_video[wp].reset (
 814                 new PlayerVideo (
 815                         video.image,
 816                         piece->content->video->crop (),
 817                         piece->content->video->fade (_film, video.frame),
 818                         scale_for_display(piece->content->video->scaled_size(_film->frame_size()), _video_container_size, _film->frame_size()),
 819                         _video_container_size,
 820                         video.eyes,
 821                         video.part,
 822                         piece->content->video->colour_conversion(),
 823                         piece->content->video->range(),
 824                         piece->content,
 825                         video.frame,
 826                         false
 827                         )
 828                 );
 829
 830         DCPTime t = time;
 831         for (int i = 0; i < frc.repeat; ++i) {
 832                 if (t < piece->end(_film)) {
 833                         emit_video (_last_video[wp], t);
 834                 }
 835                 t += one_video_frame ();
 836         }
 837 }
 838
 839 void
 840 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 841 {
 842         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 843
 844         shared_ptr<Piece> piece = wp.lock ();
 845         if (!piece) {
 846                 return;
 847         }
 848
 849         shared_ptr<AudioContent> content = piece->content->audio;
 850         DCPOMATIC_ASSERT (content);
 851
 852         int const rfr = content->resampled_frame_rate (_film);
 853
 854         /* Compute time in the DCP */
 855         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame);
 856         LOG_DEBUG_PLAYER("Received audio frame %1 at %2", content_audio.frame, to_string(time));
 857
 858         /* And the end of this block in the DCP */
 859         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), rfr);
 860
 861         /* Remove anything that comes before the start or after the end of the content */
 862         if (time < piece->position()) {
 863                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->position());
 864                 if (!cut.first) {
 865                         /* This audio is entirely discarded */
 866                         return;
 867                 }
 868                 content_audio.audio = cut.first;
 869                 time = cut.second;
 870         } else if (time > piece->end(_film)) {
 871                 /* Discard it all */
 872                 return;
 873         } else if (end > piece->end(_film)) {
 874                 Frame const remaining_frames = DCPTime(piece->end(_film) - time).frames_round(rfr);
 875                 if (remaining_frames == 0) {
 876                         return;
 877                 }
 878                 content_audio.audio.reset (new AudioBuffers(content_audio.audio, remaining_frames, 0));
 879         }
 880
 881         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 882
 883         /* Gain */
 884
 885         if (content->gain() != 0) {
 886                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 887                 gain->apply_gain (content->gain ());
 888                 content_audio.audio = gain;
 889         }
 890
 891         /* Remap */
 892
 893         content_audio.audio = remap (content_audio.audio, _film->audio_channels(), stream->mapping());
 894
 895         /* Process */
 896
 897         if (_audio_processor) {
 898                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 899         }
 900
 901         /* Push */
 902
 903         _audio_merger.push (content_audio.audio, time);
 904         /* XXX: this almost certainly needs to be more efficient; perhaps pieces fill a map to find
 905          * the piece from the stream, then we can call the right piece with no loop.
 906          */
 907         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 908                 i->set_last_push_end (stream, time + DCPTime::from_frames(content_audio.audio->frames(), _film->audio_frame_rate()));
 909         }
 910 }
 911
 912 void
 913 Player::bitmap_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentBitmapText subtitle)
 914 {
 915         shared_ptr<Piece> piece = wp.lock ();
 916         shared_ptr<const TextContent> text = wc.lock ();
 917         if (!piece || !text) {
 918                 return;
 919         }
 920
 921         /* Apply content's subtitle offsets */
 922         subtitle.sub.rectangle.x += text->x_offset ();
 923         subtitle.sub.rectangle.y += text->y_offset ();
 924
 925         /* Apply a corrective translation to keep the subtitle centred after the scale that is coming up */
 926         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * ((text->x_scale() - 1) / 2);
 927         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * ((text->y_scale() - 1) / 2);
 928
 929         /* Apply content's subtitle scale */
 930         subtitle.sub.rectangle.width *= text->x_scale ();
 931         subtitle.sub.rectangle.height *= text->y_scale ();
 932
 933         PlayerText ps;
 934         shared_ptr<Image> image = subtitle.sub.image;
 935
 936         /* We will scale the subtitle up to fit _video_container_size */
 937         int const width = subtitle.sub.rectangle.width * _video_container_size.width;
 938         int const height = subtitle.sub.rectangle.height * _video_container_size.height;
 939         if (width == 0 || height == 0) {
 940                 return;
 941         }
 942
 943         dcp::Size scaled_size (width, height);
 944         ps.bitmap.push_back (BitmapText(image->scale(scaled_size, dcp::YUV_TO_RGB_REC601, image->pixel_format(), true, _fast), subtitle.sub.rectangle));
 945         DCPTime from (piece->content_time_to_dcp(subtitle.from()));
 946
 947         _active_texts[text->type()].add_from (wc, ps, from);
 948 }
 949
 950 void
 951 Player::plain_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentStringText subtitle)
 952 {
 953         shared_ptr<Piece> piece = wp.lock ();
 954         shared_ptr<const TextContent> text = wc.lock ();
 955         if (!piece || !text) {
 956                 return;
 957         }
 958
 959         PlayerText ps;
 960         DCPTime const from (piece->content_time_to_dcp(subtitle.from()));
 961
 962         if (from > piece->end(_film)) {
 963                 return;
 964         }
 965
 966         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 967                 s.set_h_position (s.h_position() + text->x_offset ());
 968                 s.set_v_position (s.v_position() + text->y_offset ());
 969                 float const xs = text->x_scale();
 970                 float const ys = text->y_scale();
 971                 float size = s.size();
 972
 973                 /* Adjust size to express the common part of the scaling;
 974                    e.g. if xs = ys = 0.5 we scale size by 2.
 975                 */
 976                 if (xs > 1e-5 && ys > 1e-5) {
 977                         size *= 1 / min (1 / xs, 1 / ys);
 978                 }
 979                 s.set_size (size);
 980
 981                 /* Then express aspect ratio changes */
 982                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 983                         s.set_aspect_adjust (xs / ys);
 984                 }
 985
 986                 s.set_in (dcp::Time(from.seconds(), 1000));
 987                 ps.string.push_back (StringText (s, text->outline_width()));
 988                 ps.add_fonts (text->fonts ());
 989         }
 990
 991         _active_texts[text->type()].add_from (wc, ps, from);
 992 }
 993
 994 void
 995 Player::subtitle_stop (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentTime to)
 996 {
 997         shared_ptr<const TextContent> text = wc.lock ();
 998         if (!text) {
 999                 return;
1000         }
1001
1002         if (!_active_texts[text->type()].have(wc)) {
1003                 return;
1004         }
1005
1006         shared_ptr<Piece> piece = wp.lock ();
1007         if (!piece) {
1008                 return;
1009         }
1010
1011         DCPTime const dcp_to = content_time_to_dcp (piece, to);
1012
1013         if (dcp_to > piece->end(_film)) {
1014                 return;
1015         }
1016
1017         pair<PlayerText, DCPTime> from = _active_texts[text->type()].add_to (wc, dcp_to);
1018
1019         bool const always = (text->type() == TEXT_OPEN_SUBTITLE && _always_burn_open_subtitles);
1020         if (text->use() && !always && !text->burn()) {
1021                 Text (from.first, text->type(), text->dcp_track().get_value_or(DCPTextTrack()), DCPTimePeriod (from.second, dcp_to));
1022         }
1023 }
1024
1025 void
1026 Player::seek (DCPTime time, bool accurate)
1027 {
1028         boost::mutex::scoped_lock lm (_mutex);
1029         LOG_DEBUG_PLAYER("Seek to %1 (%2accurate)", to_string(time), accurate ? "" : "in");
1030
1031         if (_suspended) {
1032                 /* We can't seek in this state */
1033                 return;
1034         }
1035
1036         if (_shuffler) {
1037                 _shuffler->clear ();
1038         }
1039
1040         _delay.clear ();
1041
1042         if (_audio_processor) {
1043                 _audio_processor->flush ();
1044         }
1045
1046         _audio_merger.clear ();
1047         for (int i = 0; i < TEXT_COUNT; ++i) {
1048                 _active_texts[i].clear ();
1049         }
1050
1051         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1052                 if (time < i->position()) {
1053                         /* Before; seek to the start of the content.  Even if this request is for an inaccurate seek
1054                            we must seek this (following) content accurately, otherwise when we come to the end of the current
1055                            content we may not start right at the beginning of the next, causing a gap (if the next content has
1056                            been trimmed to a point between keyframes, or something).
1057                         */
1058                         i->decoder->seek (dcp_to_content_time (i, i->position()), true);
1059                         i->done = false;
1060                 } else if (i->position() <= time && time < i->end(_film)) {
1061                         /* During; seek to position */
1062                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
1063                         i->done = false;
1064                 } else {
1065                         /* After; this piece is done */
1066                         i->done = true;
1067                 }
1068         }
1069
1070         if (accurate) {
1071                 _last_video_time = time;
1072                 _last_video_eyes = EYES_LEFT;
1073                 _last_audio_time = time;
1074         } else {
1075                 _last_video_time = optional<DCPTime>();
1076                 _last_video_eyes = optional<Eyes>();
1077                 _last_audio_time = optional<DCPTime>();
1078         }
1079
1080         _black.set_position (time);
1081         _silent.set_position (time);
1082
1083         _last_video.clear ();
1084 }
1085
1086 void
1087 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1088 {
1089         /* We need a delay to give a little wiggle room to ensure that relevent subtitles arrive at the
1090            player before the video that requires them.
1091         */
1092         _delay.push_back (make_pair (pv, time));
1093
1094         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1095                 _last_video_time = time + one_video_frame();
1096         }
1097         _last_video_eyes = increment_eyes (pv->eyes());
1098
1099         if (_delay.size() < 3) {
1100                 return;
1101         }
1102
1103         pair<shared_ptr<PlayerVideo>, DCPTime> to_do = _delay.front();
1104         _delay.pop_front();
1105         do_emit_video (to_do.first, to_do.second);
1106 }
1107
1108 void
1109 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1110 {
1111         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1112                 for (int i = 0; i < TEXT_COUNT; ++i) {
1113                         _active_texts[i].clear_before (time);
1114                 }
1115         }
1116
1117         optional<PositionImage> subtitles = open_subtitles_for_frame (time);
1118         if (subtitles) {
1119                 pv->set_text (subtitles.get ());
1120         }
1121
1122         Video (pv, time);
1123 }
1124
1125 void
1126 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1127 {
1128         /* Log if the assert below is about to fail */
1129         if (_last_audio_time && labs(time.get() - _last_audio_time->get()) > 1) {
1130                 _film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_last_audio_time)), LogEntry::TYPE_WARNING);
1131         }
1132
1133         /* This audio must follow on from the previous, allowing for half a sample (at 48kHz) leeway */
1134         DCPOMATIC_ASSERT (!_last_audio_time || labs(time.get() - _last_audio_time->get()) < 2);
1135         Audio (data, time, _film->audio_frame_rate());
1136         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate());
1137 }
1138
1139 void
1140 Player::fill_audio (DCPTimePeriod period)
1141 {
1142         if (period.from == period.to) {
1143                 return;
1144         }
1145
1146         DCPOMATIC_ASSERT (period.from < period.to);
1147
1148         DCPTime t = period.from;
1149         while (t < period.to) {
1150                 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1151                 Frame const samples = block.frames_round(_film->audio_frame_rate());
1152                 if (samples) {
1153                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1154                         silence->make_silent ();
1155                         emit_audio (silence, t);
1156                 }
1157                 t += block;
1158         }
1159 }
1160
1161 DCPTime
1162 Player::one_video_frame () const
1163 {
1164         return DCPTime::from_frames (1, _film->video_frame_rate ());
1165 }
1166
1167 pair<shared_ptr<AudioBuffers>, DCPTime>
1168 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1169 {
1170         DCPTime const discard_time = discard_to - time;
1171         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1172         Frame remaining_frames = audio->frames() - discard_frames;
1173         if (remaining_frames <= 0) {
1174                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1175         }
1176         shared_ptr<AudioBuffers> cut (new AudioBuffers(audio, remaining_frames, discard_frames));
1177         return make_pair(cut, time + discard_time);
1178 }
1179
1180 void
1181 Player::set_dcp_decode_reduction (optional<int> reduction)
1182 {
1183         Change (CHANGE_TYPE_PENDING, PlayerProperty::DCP_DECODE_REDUCTION, false);
1184
1185         {
1186                 boost::mutex::scoped_lock lm (_mutex);
1187
1188                 if (reduction == _dcp_decode_reduction) {
1189                         lm.unlock ();
1190                         Change (CHANGE_TYPE_CANCELLED, PlayerProperty::DCP_DECODE_REDUCTION, false);
1191                         return;
1192                 }
1193
1194                 _dcp_decode_reduction = reduction;
1195                 setup_pieces_unlocked ();
1196         }
1197
1198         Change (CHANGE_TYPE_DONE, PlayerProperty::DCP_DECODE_REDUCTION, false);
1199 }
1200
1201 optional<DCPTime>
1202 Player::content_time_to_dcp (shared_ptr<Content> content, ContentTime t)
1203 {
1204         boost::mutex::scoped_lock lm (_mutex);
1205
1206         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1207                 if (i->content == content) {
1208                         return content_time_to_dcp (i, t);
1209                 }
1210         }
1211
1212         /* We couldn't find this content; perhaps things are being changed over */
1213         return optional<DCPTime>();
1214 }
1215
1216
1217 shared_ptr<const Playlist>
1218 Player::playlist () const
1219 {
1220         return _playlist ? _playlist : _film->playlist();
1221 }
1222
1223
1224 void
1225 Player::atmos (weak_ptr<Piece>, ContentAtmos data)
1226 {
1227         Atmos (data.data, DCPTime::from_frames(data.frame, _film->video_frame_rate()), data.metadata);
1228 }
1229