src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2019 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_text.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "text_content.h"
  44 #include "text_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "dcp_decoder.h"
  48 #include "image_decoder.h"
  49 #include "compose.hpp"
  50 #include "shuffler.h"
  51 #include "timer.h"
  52 #include <dcp/reel.h>
  53 #include <dcp/reel_sound_asset.h>
  54 #include <dcp/reel_subtitle_asset.h>
  55 #include <dcp/reel_picture_asset.h>
  56 #include <dcp/reel_closed_caption_asset.h>
  57 #include <boost/foreach.hpp>
  58 #include <stdint.h>
  59 #include <algorithm>
  60 #include <iostream>
  61
  62 #include "i18n.h"
  63
  64 using std::list;
  65 using std::cout;
  66 using std::min;
  67 using std::max;
  68 using std::min;
  69 using std::vector;
  70 using std::pair;
  71 using std::map;
  72 using std::make_pair;
  73 using std::copy;
  74 using boost::shared_ptr;
  75 using boost::weak_ptr;
  76 using boost::dynamic_pointer_cast;
  77 using boost::optional;
  78 using boost::scoped_ptr;
  79 using namespace dcpomatic;
  80
  81 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
  82 int const PlayerProperty::PLAYLIST = 701;
  83 int const PlayerProperty::FILM_CONTAINER = 702;
  84 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
  85 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
  86
  87 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  88         : _film (film)
  89         , _playlist (playlist)
  90         , _suspended (false)
  91         , _ignore_video (false)
  92         , _ignore_audio (false)
  93         , _ignore_text (false)
  94         , _always_burn_open_subtitles (false)
  95         , _fast (false)
  96         , _tolerant (film->tolerant())
  97         , _play_referenced (false)
  98         , _audio_merger (_film->audio_frame_rate())
  99         , _shuffler (0)
 100 {
 101         _film_changed_connection = _film->Change.connect (bind (&Player::film_change, this, _1, _2));
 102         /* The butler must hear about this first, so since we are proxying this through to the butler we must
 103            be first.
 104         */
 105         _playlist_change_connection = _playlist->Change.connect (bind (&Player::playlist_change, this, _1), boost::signals2::at_front);
 106         _playlist_content_change_connection = _playlist->ContentChange.connect (bind(&Player::playlist_content_change, this, _1, _3, _4));
 107         set_video_container_size (_film->frame_size ());
 108
 109         film_change (CHANGE_TYPE_DONE, Film::AUDIO_PROCESSOR);
 110
 111         setup_pieces ();
 112         seek (DCPTime (), true);
 113 }
 114
 115 Player::~Player ()
 116 {
 117         delete _shuffler;
 118 }
 119
 120 void
 121 Player::setup_pieces ()
 122 {
 123         boost::mutex::scoped_lock lm (_mutex);
 124         setup_pieces_unlocked ();
 125 }
 126
 127 bool
 128 have_video (shared_ptr<Piece> piece)
 129 {
 130         return piece->decoder && piece->decoder->video;
 131 }
 132
 133 bool
 134 have_audio (shared_ptr<Piece> piece)
 135 {
 136         return piece->decoder && piece->decoder->audio;
 137 }
 138
 139 void
 140 Player::setup_pieces_unlocked ()
 141 {
 142         list<shared_ptr<Piece> > old_pieces = _pieces;
 143         _pieces.clear ();
 144
 145         delete _shuffler;
 146         _shuffler = new Shuffler();
 147         _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
 148
 149         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 150
 151                 if (!i->paths_valid ()) {
 152                         continue;
 153                 }
 154
 155                 if (_ignore_video && _ignore_audio && i->text.empty()) {
 156                         /* We're only interested in text and this content has none */
 157                         continue;
 158                 }
 159
 160                 shared_ptr<Decoder> old_decoder;
 161                 BOOST_FOREACH (shared_ptr<Piece> j, old_pieces) {
 162                         if (j->content == i) {
 163                                 old_decoder = j->decoder;
 164                                 break;
 165                         }
 166                 }
 167
 168                 shared_ptr<Decoder> decoder = decoder_factory (_film, i, _fast, _tolerant, old_decoder);
 169                 FrameRateChange frc (_film, i);
 170
 171                 if (!decoder) {
 172                         /* Not something that we can decode; e.g. Atmos content */
 173                         continue;
 174                 }
 175
 176                 if (decoder->video && _ignore_video) {
 177                         decoder->video->set_ignore (true);
 178                 }
 179
 180                 if (decoder->audio && _ignore_audio) {
 181                         decoder->audio->set_ignore (true);
 182                 }
 183
 184                 if (_ignore_text) {
 185                         BOOST_FOREACH (shared_ptr<TextDecoder> i, decoder->text) {
 186                                 i->set_ignore (true);
 187                         }
 188                 }
 189
 190                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 191                 if (dcp) {
 192                         dcp->set_decode_referenced (_play_referenced);
 193                         if (_play_referenced) {
 194                                 dcp->set_forced_reduction (_dcp_decode_reduction);
 195                         }
 196                 }
 197
 198                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 199                 _pieces.push_back (piece);
 200
 201                 if (decoder->video) {
 202                         if (i->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT || i->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) {
 203                                 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
 204                                 decoder->video->Data.connect (bind (&Shuffler::video, _shuffler, weak_ptr<Piece>(piece), _1));
 205                         } else {
 206                                 decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece>(piece), _1));
 207                         }
 208                 }
 209
 210                 if (decoder->audio) {
 211                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 212                 }
 213
 214                 list<shared_ptr<TextDecoder> >::const_iterator j = decoder->text.begin();
 215
 216                 while (j != decoder->text.end()) {
 217                         (*j)->BitmapStart.connect (
 218                                 bind(&Player::bitmap_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 219                                 );
 220                         (*j)->PlainStart.connect (
 221                                 bind(&Player::plain_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 222                                 );
 223                         (*j)->Stop.connect (
 224                                 bind(&Player::subtitle_stop, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 225                                 );
 226
 227                         ++j;
 228                 }
 229         }
 230
 231         _stream_states.clear ();
 232         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 233                 if (i->content->audio) {
 234                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 235                                 _stream_states[j] = StreamState (i, i->content->position ());
 236                         }
 237                 }
 238         }
 239
 240         _black = Empty (_film, _pieces, bind(&have_video, _1));
 241         _silent = Empty (_film, _pieces, bind(&have_audio, _1));
 242
 243         _last_video_time = DCPTime ();
 244         _last_video_eyes = EYES_BOTH;
 245         _last_audio_time = DCPTime ();
 246 }
 247
 248 void
 249 Player::playlist_content_change (ChangeType type, int property, bool frequent)
 250 {
 251         if (type == CHANGE_TYPE_PENDING) {
 252                 boost::mutex::scoped_lock lm (_mutex);
 253                 /* The player content is probably about to change, so we can't carry on
 254                    until that has happened and we've rebuilt our pieces.  Stop pass()
 255                    and seek() from working until then.
 256                 */
 257                 _suspended = true;
 258         } else if (type == CHANGE_TYPE_DONE) {
 259                 /* A change in our content has gone through.  Re-build our pieces. */
 260                 setup_pieces ();
 261                 _suspended = false;
 262         } else if (type == CHANGE_TYPE_CANCELLED) {
 263                 boost::mutex::scoped_lock lm (_mutex);
 264                 _suspended = false;
 265         }
 266
 267         Change (type, property, frequent);
 268 }
 269
 270 void
 271 Player::set_video_container_size (dcp::Size s)
 272 {
 273         Change (CHANGE_TYPE_PENDING, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 274
 275         {
 276                 boost::mutex::scoped_lock lm (_mutex);
 277
 278                 if (s == _video_container_size) {
 279                         lm.unlock ();
 280                         Change (CHANGE_TYPE_CANCELLED, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 281                         return;
 282                 }
 283
 284                 _video_container_size = s;
 285
 286                 _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 287                 _black_image->make_black ();
 288         }
 289
 290         Change (CHANGE_TYPE_DONE, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 291 }
 292
 293 void
 294 Player::playlist_change (ChangeType type)
 295 {
 296         if (type == CHANGE_TYPE_DONE) {
 297                 setup_pieces ();
 298         }
 299         Change (type, PlayerProperty::PLAYLIST, false);
 300 }
 301
 302 void
 303 Player::film_change (ChangeType type, Film::Property p)
 304 {
 305         /* Here we should notice Film properties that affect our output, and
 306            alert listeners that our output now would be different to how it was
 307            last time we were run.
 308         */
 309
 310         if (p == Film::CONTAINER) {
 311                 Change (type, PlayerProperty::FILM_CONTAINER, false);
 312         } else if (p == Film::VIDEO_FRAME_RATE) {
 313                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 314                    so we need new pieces here.
 315                 */
 316                 if (type == CHANGE_TYPE_DONE) {
 317                         setup_pieces ();
 318                 }
 319                 Change (type, PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
 320         } else if (p == Film::AUDIO_PROCESSOR) {
 321                 if (type == CHANGE_TYPE_DONE && _film->audio_processor ()) {
 322                         boost::mutex::scoped_lock lm (_mutex);
 323                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 324                 }
 325         } else if (p == Film::AUDIO_CHANNELS) {
 326                 if (type == CHANGE_TYPE_DONE) {
 327                         boost::mutex::scoped_lock lm (_mutex);
 328                         _audio_merger.clear ();
 329                 }
 330         }
 331 }
 332
 333 shared_ptr<PlayerVideo>
 334 Player::black_player_video_frame (Eyes eyes) const
 335 {
 336         return shared_ptr<PlayerVideo> (
 337                 new PlayerVideo (
 338                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 339                         Crop (),
 340                         optional<double> (),
 341                         _video_container_size,
 342                         _video_container_size,
 343                         eyes,
 344                         PART_WHOLE,
 345                         PresetColourConversion::all().front().conversion,
 346                         VIDEO_RANGE_FULL,
 347                         boost::weak_ptr<Content>(),
 348                         boost::optional<Frame>()
 349                 )
 350         );
 351 }
 352
 353 Frame
 354 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 355 {
 356         DCPTime s = t - piece->content->position ();
 357         s = min (piece->content->length_after_trim(_film), s);
 358         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 359
 360         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 361            then convert that ContentTime to frames at the content's rate.  However this fails for
 362            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 363            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 364
 365            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 366         */
 367         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 368 }
 369
 370 DCPTime
 371 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 372 {
 373         /* See comment in dcp_to_content_video */
 374         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime(piece->content->trim_start(), piece->frc);
 375         return d + piece->content->position();
 376 }
 377
 378 Frame
 379 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 380 {
 381         DCPTime s = t - piece->content->position ();
 382         s = min (piece->content->length_after_trim(_film), s);
 383         /* See notes in dcp_to_content_video */
 384         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 385 }
 386
 387 DCPTime
 388 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 389 {
 390         /* See comment in dcp_to_content_video */
 391         return DCPTime::from_frames (f, _film->audio_frame_rate())
 392                 - DCPTime (piece->content->trim_start(), piece->frc)
 393                 + piece->content->position();
 394 }
 395
 396 ContentTime
 397 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 398 {
 399         DCPTime s = t - piece->content->position ();
 400         s = min (piece->content->length_after_trim(_film), s);
 401         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 402 }
 403
 404 DCPTime
 405 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 406 {
 407         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 408 }
 409
 410 list<shared_ptr<Font> >
 411 Player::get_subtitle_fonts ()
 412 {
 413         boost::mutex::scoped_lock lm (_mutex);
 414
 415         list<shared_ptr<Font> > fonts;
 416         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 417                 BOOST_FOREACH (shared_ptr<TextContent> j, i->content->text) {
 418                         /* XXX: things may go wrong if there are duplicate font IDs
 419                            with different font files.
 420                         */
 421                         list<shared_ptr<Font> > f = j->fonts ();
 422                         copy (f.begin(), f.end(), back_inserter (fonts));
 423                 }
 424         }
 425
 426         return fonts;
 427 }
 428
 429 /** Set this player never to produce any video data */
 430 void
 431 Player::set_ignore_video ()
 432 {
 433         boost::mutex::scoped_lock lm (_mutex);
 434         _ignore_video = true;
 435         setup_pieces_unlocked ();
 436 }
 437
 438 void
 439 Player::set_ignore_audio ()
 440 {
 441         boost::mutex::scoped_lock lm (_mutex);
 442         _ignore_audio = true;
 443         setup_pieces_unlocked ();
 444 }
 445
 446 void
 447 Player::set_ignore_text ()
 448 {
 449         boost::mutex::scoped_lock lm (_mutex);
 450         _ignore_text = true;
 451         setup_pieces_unlocked ();
 452 }
 453
 454 /** Set the player to always burn open texts into the image regardless of the content settings */
 455 void
 456 Player::set_always_burn_open_subtitles ()
 457 {
 458         boost::mutex::scoped_lock lm (_mutex);
 459         _always_burn_open_subtitles = true;
 460 }
 461
 462 /** Sets up the player to be faster, possibly at the expense of quality */
 463 void
 464 Player::set_fast ()
 465 {
 466         boost::mutex::scoped_lock lm (_mutex);
 467         _fast = true;
 468         setup_pieces_unlocked ();
 469 }
 470
 471 void
 472 Player::set_play_referenced ()
 473 {
 474         boost::mutex::scoped_lock lm (_mutex);
 475         _play_referenced = true;
 476         setup_pieces_unlocked ();
 477 }
 478
 479 static void
 480 maybe_add_asset (list<ReferencedReelAsset>& a, shared_ptr<dcp::ReelAsset> r, Frame reel_trim_start, Frame reel_trim_end, DCPTime from, int const ffr)
 481 {
 482         DCPOMATIC_ASSERT (r);
 483         r->set_entry_point (r->entry_point().get_value_or(0) + reel_trim_start);
 484         r->set_duration (r->actual_duration() - reel_trim_start - reel_trim_end);
 485         if (r->actual_duration() > 0) {
 486                 a.push_back (
 487                         ReferencedReelAsset(r, DCPTimePeriod(from, from + DCPTime::from_frames(r->actual_duration(), ffr)))
 488                         );
 489         }
 490 }
 491
 492 list<ReferencedReelAsset>
 493 Player::get_reel_assets ()
 494 {
 495         /* Does not require a lock on _mutex as it's only called from DCPEncoder */
 496
 497         list<ReferencedReelAsset> a;
 498
 499         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 500                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 501                 if (!j) {
 502                         continue;
 503                 }
 504
 505                 scoped_ptr<DCPDecoder> decoder;
 506                 try {
 507                         decoder.reset (new DCPDecoder (_film, j, false, false, shared_ptr<DCPDecoder>()));
 508                 } catch (...) {
 509                         return a;
 510                 }
 511
 512                 DCPOMATIC_ASSERT (j->video_frame_rate ());
 513                 double const cfr = j->video_frame_rate().get();
 514                 Frame const trim_start = j->trim_start().frames_round (cfr);
 515                 Frame const trim_end = j->trim_end().frames_round (cfr);
 516                 int const ffr = _film->video_frame_rate ();
 517
 518                 /* position in the asset from the start */
 519                 int64_t offset_from_start = 0;
 520                 /* position in the asset from the end */
 521                 int64_t offset_from_end = 0;
 522                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 523                         /* Assume that main picture duration is the length of the reel */
 524                         offset_from_end += k->main_picture()->actual_duration();
 525                 }
 526
 527                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 528
 529                         /* Assume that main picture duration is the length of the reel */
 530                         int64_t const reel_duration = k->main_picture()->actual_duration();
 531
 532                         /* See doc/design/trim_reels.svg */
 533                         Frame const reel_trim_start = min(reel_duration, max(int64_t(0), trim_start - offset_from_start));
 534                         Frame const reel_trim_end =   min(reel_duration, max(int64_t(0), reel_duration - (offset_from_end - trim_end)));
 535
 536                         DCPTime const from = i->position() + DCPTime::from_frames (offset_from_start, _film->video_frame_rate());
 537                         if (j->reference_video ()) {
 538                                 maybe_add_asset (a, k->main_picture(), reel_trim_start, reel_trim_end, from, ffr);
 539                         }
 540
 541                         if (j->reference_audio ()) {
 542                                 maybe_add_asset (a, k->main_sound(), reel_trim_start, reel_trim_end, from, ffr);
 543                         }
 544
 545                         if (j->reference_text (TEXT_OPEN_SUBTITLE)) {
 546                                 maybe_add_asset (a, k->main_subtitle(), reel_trim_start, reel_trim_end, from, ffr);
 547                         }
 548
 549                         if (j->reference_text (TEXT_CLOSED_CAPTION)) {
 550                                 BOOST_FOREACH (shared_ptr<dcp::ReelClosedCaptionAsset> l, k->closed_captions()) {
 551                                         maybe_add_asset (a, l, reel_trim_start, reel_trim_end, from, ffr);
 552                                 }
 553                         }
 554
 555                         offset_from_start += reel_duration;
 556                         offset_from_end -= reel_duration;
 557                 }
 558         }
 559
 560         return a;
 561 }
 562
 563 bool
 564 Player::pass ()
 565 {
 566         boost::mutex::scoped_lock lm (_mutex);
 567
 568         if (_suspended) {
 569                 /* We can't pass in this state */
 570                 return false;
 571         }
 572
 573         if (_playlist->length(_film) == DCPTime()) {
 574                 /* Special case of an empty Film; just give one black frame */
 575                 emit_video (black_player_video_frame(EYES_BOTH), DCPTime());
 576                 return true;
 577         }
 578
 579         /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
 580
 581         shared_ptr<Piece> earliest_content;
 582         optional<DCPTime> earliest_time;
 583
 584         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 585                 if (i->done) {
 586                         continue;
 587                 }
 588
 589                 DCPTime const t = content_time_to_dcp (i, max(i->decoder->position(), i->content->trim_start()));
 590                 if (t > i->content->end(_film)) {
 591                         i->done = true;
 592                 } else {
 593
 594                         /* Given two choices at the same time, pick the one with texts so we see it before
 595                            the video.
 596                         */
 597                         if (!earliest_time || t < *earliest_time || (t == *earliest_time && !i->decoder->text.empty())) {
 598                                 earliest_time = t;
 599                                 earliest_content = i;
 600                         }
 601                 }
 602         }
 603
 604         bool done = false;
 605
 606         enum {
 607                 NONE,
 608                 CONTENT,
 609                 BLACK,
 610                 SILENT
 611         } which = NONE;
 612
 613         if (earliest_content) {
 614                 which = CONTENT;
 615         }
 616
 617         if (!_black.done() && !_ignore_video && (!earliest_time || _black.position() < *earliest_time)) {
 618                 earliest_time = _black.position ();
 619                 which = BLACK;
 620         }
 621
 622         if (!_silent.done() && !_ignore_audio && (!earliest_time || _silent.position() < *earliest_time)) {
 623                 earliest_time = _silent.position ();
 624                 which = SILENT;
 625         }
 626
 627         switch (which) {
 628         case CONTENT:
 629         {
 630                 earliest_content->done = earliest_content->decoder->pass ();
 631                 shared_ptr<DCPContent> dcp = dynamic_pointer_cast<DCPContent>(earliest_content->content);
 632                 if (dcp && !_play_referenced && dcp->reference_audio()) {
 633                         /* We are skipping some referenced DCP audio content, so we need to update _last_audio_time
 634                            to `hide' the fact that no audio was emitted during the referenced DCP (though
 635                            we need to behave as though it was).
 636                         */
 637                         _last_audio_time = dcp->end (_film);
 638                 }
 639                 break;
 640         }
 641         case BLACK:
 642                 emit_video (black_player_video_frame(EYES_BOTH), _black.position());
 643                 _black.set_position (_black.position() + one_video_frame());
 644                 break;
 645         case SILENT:
 646         {
 647                 DCPTimePeriod period (_silent.period_at_position());
 648                 if (_last_audio_time) {
 649                         /* Sometimes the thing that happened last finishes fractionally before
 650                            or after this silence.  Bodge the start time of the silence to fix it.
 651                            I think is nothing too bad to worry about since we will just add or
 652                            remove a little silence at the end of some content.
 653                         */
 654                         int64_t const error = labs(period.from.get() - _last_audio_time->get());
 655                         int64_t const too_much_error = 4;
 656                         if (error >= too_much_error) {
 657                                 _film->log()->log(String::compose("Silence starting before or after last audio by %1", error), LogEntry::TYPE_ERROR);
 658                         }
 659                         DCPOMATIC_ASSERT (error < too_much_error);
 660                         period.from = *_last_audio_time;
 661                 }
 662                 if (period.duration() > one_video_frame()) {
 663                         period.to = period.from + one_video_frame();
 664                 }
 665                 fill_audio (period);
 666                 _silent.set_position (period.to);
 667                 break;
 668         }
 669         case NONE:
 670                 done = true;
 671                 break;
 672         }
 673
 674         /* Emit any audio that is ready */
 675
 676         /* Work out the time before which the audio is definitely all here.  This is the earliest last_push_end of one
 677            of our streams, or the position of the _silent.
 678         */
 679         DCPTime pull_to = _film->length ();
 680         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 681                 if (!i->second.piece->done && i->second.last_push_end < pull_to) {
 682                         pull_to = i->second.last_push_end;
 683                 }
 684         }
 685         if (!_silent.done() && _silent.position() < pull_to) {
 686                 pull_to = _silent.position();
 687         }
 688
 689         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 690         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 691                 if (_last_audio_time && i->second < *_last_audio_time) {
 692                         /* This new data comes before the last we emitted (or the last seek); discard it */
 693                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 694                         if (!cut.first) {
 695                                 continue;
 696                         }
 697                         *i = cut;
 698                 } else if (_last_audio_time && i->second > *_last_audio_time) {
 699                         /* There's a gap between this data and the last we emitted; fill with silence */
 700                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 701                 }
 702
 703                 emit_audio (i->first, i->second);
 704         }
 705
 706         if (done) {
 707                 _shuffler->flush ();
 708                 for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 709                         do_emit_video(i->first, i->second);
 710                 }
 711         }
 712
 713         return done;
 714 }
 715
 716 /** @return Open subtitles for the frame at the given time, converted to images */
 717 optional<PositionImage>
 718 Player::open_subtitles_for_frame (DCPTime time) const
 719 {
 720         list<PositionImage> captions;
 721         int const vfr = _film->video_frame_rate();
 722
 723         BOOST_FOREACH (
 724                 PlayerText j,
 725                 _active_texts[TEXT_OPEN_SUBTITLE].get_burnt(DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), _always_burn_open_subtitles)
 726                 ) {
 727
 728                 /* Bitmap subtitles */
 729                 BOOST_FOREACH (BitmapText i, j.bitmap) {
 730                         if (!i.image) {
 731                                 continue;
 732                         }
 733
 734                         /* i.image will already have been scaled to fit _video_container_size */
 735                         dcp::Size scaled_size (i.rectangle.width * _video_container_size.width, i.rectangle.height * _video_container_size.height);
 736
 737                         captions.push_back (
 738                                 PositionImage (
 739                                         i.image,
 740                                         Position<int> (
 741                                                 lrint (_video_container_size.width * i.rectangle.x),
 742                                                 lrint (_video_container_size.height * i.rectangle.y)
 743                                                 )
 744                                         )
 745                                 );
 746                 }
 747
 748                 /* String subtitles (rendered to an image) */
 749                 if (!j.string.empty ()) {
 750                         list<PositionImage> s = render_text (j.string, j.fonts, _video_container_size, time, vfr);
 751                         copy (s.begin(), s.end(), back_inserter (captions));
 752                 }
 753         }
 754
 755         if (captions.empty ()) {
 756                 return optional<PositionImage> ();
 757         }
 758
 759         return merge (captions);
 760 }
 761
 762 void
 763 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 764 {
 765         shared_ptr<Piece> piece = wp.lock ();
 766         if (!piece) {
 767                 return;
 768         }
 769
 770         FrameRateChange frc (_film, piece->content);
 771         if (frc.skip && (video.frame % 2) == 1) {
 772                 return;
 773         }
 774
 775         /* Time of the first frame we will emit */
 776         DCPTime const time = content_video_to_dcp (piece, video.frame);
 777
 778         /* Discard if it's before the content's period or the last accurate seek.  We can't discard
 779            if it's after the content's period here as in that case we still need to fill any gap between
 780            `now' and the end of the content's period.
 781         */
 782         if (time < piece->content->position() || (_last_video_time && time < *_last_video_time)) {
 783                 return;
 784         }
 785
 786         /* Fill gaps that we discover now that we have some video which needs to be emitted.
 787            This is where we need to fill to.
 788         */
 789         DCPTime fill_to = min (time, piece->content->end(_film));
 790
 791         if (_last_video_time) {
 792                 DCPTime fill_from = max (*_last_video_time, piece->content->position());
 793
 794                 /* Fill if we have more than half a frame to do */
 795                 if ((fill_to - fill_from) > one_video_frame() / 2) {
 796                         LastVideoMap::const_iterator last = _last_video.find (wp);
 797                         if (_film->three_d()) {
 798                                 Eyes fill_to_eyes = video.eyes;
 799                                 if (fill_to_eyes == EYES_BOTH) {
 800                                         fill_to_eyes = EYES_LEFT;
 801                                 }
 802                                 if (fill_to == piece->content->end(_film)) {
 803                                         /* Don't fill after the end of the content */
 804                                         fill_to_eyes = EYES_LEFT;
 805                                 }
 806                                 DCPTime j = fill_from;
 807                                 Eyes eyes = _last_video_eyes.get_value_or(EYES_LEFT);
 808                                 if (eyes == EYES_BOTH) {
 809                                         eyes = EYES_LEFT;
 810                                 }
 811                                 while (j < fill_to || eyes != fill_to_eyes) {
 812                                         if (last != _last_video.end()) {
 813                                                 shared_ptr<PlayerVideo> copy = last->second->shallow_copy();
 814                                                 copy->set_eyes (eyes);
 815                                                 emit_video (copy, j);
 816                                         } else {
 817                                                 emit_video (black_player_video_frame(eyes), j);
 818                                         }
 819                                         if (eyes == EYES_RIGHT) {
 820                                                 j += one_video_frame();
 821                                         }
 822                                         eyes = increment_eyes (eyes);
 823                                 }
 824                         } else {
 825                                 for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
 826                                         if (last != _last_video.end()) {
 827                                                 emit_video (last->second, j);
 828                                         } else {
 829                                                 emit_video (black_player_video_frame(EYES_BOTH), j);
 830                                         }
 831                                 }
 832                         }
 833                 }
 834         }
 835
 836         _last_video[wp].reset (
 837                 new PlayerVideo (
 838                         video.image,
 839                         piece->content->video->crop (),
 840                         piece->content->video->fade (_film, video.frame),
 841                         piece->content->video->scale().size (
 842                                 piece->content->video, _video_container_size, _film->frame_size ()
 843                                 ),
 844                         _video_container_size,
 845                         video.eyes,
 846                         video.part,
 847                         piece->content->video->colour_conversion(),
 848                         piece->content->video->range(),
 849                         piece->content,
 850                         video.frame
 851                         )
 852                 );
 853
 854         DCPTime t = time;
 855         for (int i = 0; i < frc.repeat; ++i) {
 856                 if (t < piece->content->end(_film)) {
 857                         emit_video (_last_video[wp], t);
 858                 }
 859                 t += one_video_frame ();
 860         }
 861 }
 862
 863 void
 864 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 865 {
 866         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 867
 868         shared_ptr<Piece> piece = wp.lock ();
 869         if (!piece) {
 870                 return;
 871         }
 872
 873         shared_ptr<AudioContent> content = piece->content->audio;
 874         DCPOMATIC_ASSERT (content);
 875
 876         int const rfr = content->resampled_frame_rate (_film);
 877
 878         /* Compute time in the DCP */
 879         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame);
 880         /* And the end of this block in the DCP */
 881         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), rfr);
 882
 883         /* Remove anything that comes before the start or after the end of the content */
 884         if (time < piece->content->position()) {
 885                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 886                 if (!cut.first) {
 887                         /* This audio is entirely discarded */
 888                         return;
 889                 }
 890                 content_audio.audio = cut.first;
 891                 time = cut.second;
 892         } else if (time > piece->content->end(_film)) {
 893                 /* Discard it all */
 894                 return;
 895         } else if (end > piece->content->end(_film)) {
 896                 Frame const remaining_frames = DCPTime(piece->content->end(_film) - time).frames_round(rfr);
 897                 if (remaining_frames == 0) {
 898                         return;
 899                 }
 900                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 901                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 902                 content_audio.audio = cut;
 903         }
 904
 905         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 906
 907         /* Gain */
 908
 909         if (content->gain() != 0) {
 910                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 911                 gain->apply_gain (content->gain ());
 912                 content_audio.audio = gain;
 913         }
 914
 915         /* Remap */
 916
 917         content_audio.audio = remap (content_audio.audio, _film->audio_channels(), stream->mapping());
 918
 919         /* Process */
 920
 921         if (_audio_processor) {
 922                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 923         }
 924
 925         /* Push */
 926
 927         _audio_merger.push (content_audio.audio, time);
 928         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 929         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 930 }
 931
 932 void
 933 Player::bitmap_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentBitmapText subtitle)
 934 {
 935         shared_ptr<Piece> piece = wp.lock ();
 936         shared_ptr<const TextContent> text = wc.lock ();
 937         if (!piece || !text) {
 938                 return;
 939         }
 940
 941         /* Apply content's subtitle offsets */
 942         subtitle.sub.rectangle.x += text->x_offset ();
 943         subtitle.sub.rectangle.y += text->y_offset ();
 944
 945         /* Apply a corrective translation to keep the subtitle centred after the scale that is coming up */
 946         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * ((text->x_scale() - 1) / 2);
 947         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * ((text->y_scale() - 1) / 2);
 948
 949         /* Apply content's subtitle scale */
 950         subtitle.sub.rectangle.width *= text->x_scale ();
 951         subtitle.sub.rectangle.height *= text->y_scale ();
 952
 953         PlayerText ps;
 954         shared_ptr<Image> image = subtitle.sub.image;
 955         /* We will scale the subtitle up to fit _video_container_size */
 956         dcp::Size scaled_size (subtitle.sub.rectangle.width * _video_container_size.width, subtitle.sub.rectangle.height * _video_container_size.height);
 957         ps.bitmap.push_back (BitmapText(image->scale(scaled_size, dcp::YUV_TO_RGB_REC601, image->pixel_format(), true, _fast), subtitle.sub.rectangle));
 958         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 959
 960         _active_texts[text->type()].add_from (wc, ps, from);
 961 }
 962
 963 void
 964 Player::plain_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentStringText subtitle)
 965 {
 966         shared_ptr<Piece> piece = wp.lock ();
 967         shared_ptr<const TextContent> text = wc.lock ();
 968         if (!piece || !text) {
 969                 return;
 970         }
 971
 972         PlayerText ps;
 973         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 974
 975         if (from > piece->content->end(_film)) {
 976                 return;
 977         }
 978
 979         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 980                 s.set_h_position (s.h_position() + text->x_offset ());
 981                 s.set_v_position (s.v_position() + text->y_offset ());
 982                 float const xs = text->x_scale();
 983                 float const ys = text->y_scale();
 984                 float size = s.size();
 985
 986                 /* Adjust size to express the common part of the scaling;
 987                    e.g. if xs = ys = 0.5 we scale size by 2.
 988                 */
 989                 if (xs > 1e-5 && ys > 1e-5) {
 990                         size *= 1 / min (1 / xs, 1 / ys);
 991                 }
 992                 s.set_size (size);
 993
 994                 /* Then express aspect ratio changes */
 995                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 996                         s.set_aspect_adjust (xs / ys);
 997                 }
 998
 999                 s.set_in (dcp::Time(from.seconds(), 1000));
1000                 ps.string.push_back (StringText (s, text->outline_width()));
1001                 ps.add_fonts (text->fonts ());
1002         }
1003
1004         _active_texts[text->type()].add_from (wc, ps, from);
1005 }
1006
1007 void
1008 Player::subtitle_stop (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentTime to)
1009 {
1010         shared_ptr<const TextContent> text = wc.lock ();
1011         if (!text) {
1012                 return;
1013         }
1014
1015         if (!_active_texts[text->type()].have(wc)) {
1016                 return;
1017         }
1018
1019         shared_ptr<Piece> piece = wp.lock ();
1020         if (!piece) {
1021                 return;
1022         }
1023
1024         DCPTime const dcp_to = content_time_to_dcp (piece, to);
1025
1026         if (dcp_to > piece->content->end(_film)) {
1027                 return;
1028         }
1029
1030         pair<PlayerText, DCPTime> from = _active_texts[text->type()].add_to (wc, dcp_to);
1031
1032         bool const always = (text->type() == TEXT_OPEN_SUBTITLE && _always_burn_open_subtitles);
1033         if (text->use() && !always && !text->burn()) {
1034                 Text (from.first, text->type(), text->dcp_track().get_value_or(DCPTextTrack()), DCPTimePeriod (from.second, dcp_to));
1035         }
1036 }
1037
1038 void
1039 Player::seek (DCPTime time, bool accurate)
1040 {
1041         boost::mutex::scoped_lock lm (_mutex);
1042
1043         if (_suspended) {
1044                 /* We can't seek in this state */
1045                 return;
1046         }
1047
1048         if (_shuffler) {
1049                 _shuffler->clear ();
1050         }
1051
1052         _delay.clear ();
1053
1054         if (_audio_processor) {
1055                 _audio_processor->flush ();
1056         }
1057
1058         _audio_merger.clear ();
1059         for (int i = 0; i < TEXT_COUNT; ++i) {
1060                 _active_texts[i].clear ();
1061         }
1062
1063         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1064                 if (time < i->content->position()) {
1065                         /* Before; seek to the start of the content.  Even if this request is for an inaccurate seek
1066                            we must seek this (following) content accurately, otherwise when we come to the end of the current
1067                            content we may not start right at the beginning of the next, causing a gap (if the next content has
1068                            been trimmed to a point between keyframes, or something).
1069                         */
1070                         i->decoder->seek (dcp_to_content_time (i, i->content->position()), true);
1071                         i->done = false;
1072                 } else if (i->content->position() <= time && time < i->content->end(_film)) {
1073                         /* During; seek to position */
1074                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
1075                         i->done = false;
1076                 } else {
1077                         /* After; this piece is done */
1078                         i->done = true;
1079                 }
1080         }
1081
1082         if (accurate) {
1083                 _last_video_time = time;
1084                 _last_video_eyes = EYES_LEFT;
1085                 _last_audio_time = time;
1086         } else {
1087                 _last_video_time = optional<DCPTime>();
1088                 _last_video_eyes = optional<Eyes>();
1089                 _last_audio_time = optional<DCPTime>();
1090         }
1091
1092         _black.set_position (time);
1093         _silent.set_position (time);
1094
1095         _last_video.clear ();
1096 }
1097
1098 void
1099 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1100 {
1101         /* We need a delay to give a little wiggle room to ensure that relevent subtitles arrive at the
1102            player before the video that requires them.
1103         */
1104         _delay.push_back (make_pair (pv, time));
1105
1106         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1107                 _last_video_time = time + one_video_frame();
1108         }
1109         _last_video_eyes = increment_eyes (pv->eyes());
1110
1111         if (_delay.size() < 3) {
1112                 return;
1113         }
1114
1115         pair<shared_ptr<PlayerVideo>, DCPTime> to_do = _delay.front();
1116         _delay.pop_front();
1117         do_emit_video (to_do.first, to_do.second);
1118 }
1119
1120 void
1121 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1122 {
1123         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1124                 for (int i = 0; i < TEXT_COUNT; ++i) {
1125                         _active_texts[i].clear_before (time);
1126                 }
1127         }
1128
1129         optional<PositionImage> subtitles = open_subtitles_for_frame (time);
1130         if (subtitles) {
1131                 pv->set_text (subtitles.get ());
1132         }
1133
1134         Video (pv, time);
1135 }
1136
1137 void
1138 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1139 {
1140         /* Log if the assert below is about to fail */
1141         if (_last_audio_time && labs(time.get() - _last_audio_time->get()) > 1) {
1142                 _film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_last_audio_time)), LogEntry::TYPE_WARNING);
1143         }
1144
1145         /* This audio must follow on from the previous, allowing for half a sample (at 48kHz) leeway */
1146         DCPOMATIC_ASSERT (!_last_audio_time || labs(time.get() - _last_audio_time->get()) < 2);
1147         Audio (data, time, _film->audio_frame_rate());
1148         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate());
1149 }
1150
1151 void
1152 Player::fill_audio (DCPTimePeriod period)
1153 {
1154         if (period.from == period.to) {
1155                 return;
1156         }
1157
1158         DCPOMATIC_ASSERT (period.from < period.to);
1159
1160         DCPTime t = period.from;
1161         while (t < period.to) {
1162                 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1163                 Frame const samples = block.frames_round(_film->audio_frame_rate());
1164                 if (samples) {
1165                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1166                         silence->make_silent ();
1167                         emit_audio (silence, t);
1168                 }
1169                 t += block;
1170         }
1171 }
1172
1173 DCPTime
1174 Player::one_video_frame () const
1175 {
1176         return DCPTime::from_frames (1, _film->video_frame_rate ());
1177 }
1178
1179 pair<shared_ptr<AudioBuffers>, DCPTime>
1180 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1181 {
1182         DCPTime const discard_time = discard_to - time;
1183         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1184         Frame remaining_frames = audio->frames() - discard_frames;
1185         if (remaining_frames <= 0) {
1186                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1187         }
1188         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1189         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1190         return make_pair(cut, time + discard_time);
1191 }
1192
1193 void
1194 Player::set_dcp_decode_reduction (optional<int> reduction)
1195 {
1196         Change (CHANGE_TYPE_PENDING, PlayerProperty::DCP_DECODE_REDUCTION, false);
1197
1198         {
1199                 boost::mutex::scoped_lock lm (_mutex);
1200
1201                 if (reduction == _dcp_decode_reduction) {
1202                         lm.unlock ();
1203                         Change (CHANGE_TYPE_CANCELLED, PlayerProperty::DCP_DECODE_REDUCTION, false);
1204                         return;
1205                 }
1206
1207                 _dcp_decode_reduction = reduction;
1208                 setup_pieces_unlocked ();
1209         }
1210
1211         Change (CHANGE_TYPE_DONE, PlayerProperty::DCP_DECODE_REDUCTION, false);
1212 }
1213
1214 optional<DCPTime>
1215 Player::content_time_to_dcp (shared_ptr<Content> content, ContentTime t)
1216 {
1217         boost::mutex::scoped_lock lm (_mutex);
1218
1219         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1220                 if (i->content == content) {
1221                         return content_time_to_dcp (i, t);
1222                 }
1223         }
1224
1225         /* We couldn't find this content; perhaps things are being changed over */
1226         return optional<DCPTime>();
1227 }