src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2020 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_text.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "text_content.h"
  44 #include "text_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "dcp_decoder.h"
  48 #include "image_decoder.h"
  49 #include "compose.hpp"
  50 #include "shuffler.h"
  51 #include <dcp/reel.h>
  52 #include <dcp/reel_sound_asset.h>
  53 #include <dcp/reel_subtitle_asset.h>
  54 #include <dcp/reel_picture_asset.h>
  55 #include <dcp/reel_closed_caption_asset.h>
  56 #include <boost/foreach.hpp>
  57 #include <stdint.h>
  58 #include <algorithm>
  59 #include <iostream>
  60
  61 #include "i18n.h"
  62
  63 using std::list;
  64 using std::cout;
  65 using std::min;
  66 using std::max;
  67 using std::min;
  68 using std::vector;
  69 using std::pair;
  70 using std::map;
  71 using std::make_pair;
  72 using std::copy;
  73 using boost::shared_ptr;
  74 using boost::weak_ptr;
  75 using boost::dynamic_pointer_cast;
  76 using boost::optional;
  77 using boost::scoped_ptr;
  78
  79 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
  80 int const PlayerProperty::PLAYLIST = 701;
  81 int const PlayerProperty::FILM_CONTAINER = 702;
  82 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
  83 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
  84
  85 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  86         : _film (film)
  87         , _playlist (playlist)
  88         , _suspended (0)
  89         , _ignore_video (false)
  90         , _ignore_audio (false)
  91         , _ignore_text (false)
  92         , _always_burn_open_subtitles (false)
  93         , _fast (false)
  94         , _play_referenced (false)
  95         , _audio_merger (_film->audio_frame_rate())
  96         , _shuffler (0)
  97 {
  98         _film_changed_connection = _film->Change.connect (bind (&Player::film_change, this, _1, _2));
  99         /* The butler must hear about this first, so since we are proxying this through to the butler we must
 100            be first.
 101         */
 102         _playlist_change_connection = _playlist->Change.connect (bind (&Player::playlist_change, this, _1), boost::signals2::at_front);
 103         _playlist_content_change_connection = _playlist->ContentChange.connect (bind(&Player::playlist_content_change, this, _1, _3, _4));
 104         set_video_container_size (_film->frame_size ());
 105
 106         film_change (CHANGE_TYPE_DONE, Film::AUDIO_PROCESSOR);
 107
 108         setup_pieces ();
 109         seek (DCPTime (), true);
 110 }
 111
 112 Player::~Player ()
 113 {
 114         delete _shuffler;
 115 }
 116
 117 void
 118 Player::setup_pieces ()
 119 {
 120         boost::mutex::scoped_lock lm (_mutex);
 121         setup_pieces_unlocked ();
 122 }
 123
 124 bool
 125 have_video (shared_ptr<Piece> piece)
 126 {
 127         return piece->decoder && piece->decoder->video;
 128 }
 129
 130 bool
 131 have_audio (shared_ptr<Piece> piece)
 132 {
 133         return piece->decoder && piece->decoder->audio;
 134 }
 135
 136 void
 137 Player::setup_pieces_unlocked ()
 138 {
 139         _pieces.clear ();
 140
 141         delete _shuffler;
 142         _shuffler = new Shuffler();
 143         _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
 144
 145         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 146
 147                 if (!i->paths_valid ()) {
 148                         continue;
 149                 }
 150
 151                 if (_ignore_video && _ignore_audio && i->text.empty()) {
 152                         /* We're only interested in text and this content has none */
 153                         continue;
 154                 }
 155
 156                 shared_ptr<Decoder> decoder = decoder_factory (_film, i, _fast);
 157                 FrameRateChange frc (_film, i);
 158
 159                 if (!decoder) {
 160                         /* Not something that we can decode; e.g. Atmos content */
 161                         continue;
 162                 }
 163
 164                 if (decoder->video && _ignore_video) {
 165                         decoder->video->set_ignore (true);
 166                 }
 167
 168                 if (decoder->audio && _ignore_audio) {
 169                         decoder->audio->set_ignore (true);
 170                 }
 171
 172                 if (_ignore_text) {
 173                         BOOST_FOREACH (shared_ptr<TextDecoder> i, decoder->text) {
 174                                 i->set_ignore (true);
 175                         }
 176                 }
 177
 178                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 179                 if (dcp) {
 180                         dcp->set_decode_referenced (_play_referenced);
 181                         if (_play_referenced) {
 182                                 dcp->set_forced_reduction (_dcp_decode_reduction);
 183                         }
 184                 }
 185
 186                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 187                 _pieces.push_back (piece);
 188
 189                 if (decoder->video) {
 190                         if (i->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT || i->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) {
 191                                 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
 192                                 decoder->video->Data.connect (bind (&Shuffler::video, _shuffler, weak_ptr<Piece>(piece), _1));
 193                         } else {
 194                                 decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece>(piece), _1));
 195                         }
 196                 }
 197
 198                 if (decoder->audio) {
 199                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 200                 }
 201
 202                 list<shared_ptr<TextDecoder> >::const_iterator j = decoder->text.begin();
 203
 204                 while (j != decoder->text.end()) {
 205                         (*j)->BitmapStart.connect (
 206                                 bind(&Player::bitmap_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 207                                 );
 208                         (*j)->PlainStart.connect (
 209                                 bind(&Player::plain_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 210                                 );
 211                         (*j)->Stop.connect (
 212                                 bind(&Player::subtitle_stop, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 213                                 );
 214
 215                         ++j;
 216                 }
 217         }
 218
 219         _stream_states.clear ();
 220         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 221                 if (i->content->audio) {
 222                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 223                                 _stream_states[j] = StreamState (i, i->content->position ());
 224                         }
 225                 }
 226         }
 227
 228         _black = Empty (_film, _pieces, bind(&have_video, _1));
 229         _silent = Empty (_film, _pieces, bind(&have_audio, _1));
 230
 231         _last_video_time = DCPTime ();
 232         _last_video_eyes = EYES_BOTH;
 233         _last_audio_time = DCPTime ();
 234 }
 235
 236 void
 237 Player::playlist_content_change (ChangeType type, int property, bool frequent)
 238 {
 239         if (type == CHANGE_TYPE_PENDING) {
 240                 /* The player content is probably about to change, so we can't carry on
 241                    until that has happened and we've rebuilt our pieces.  Stop pass()
 242                    and seek() from working until then.
 243                 */
 244                 ++_suspended;
 245         } else if (type == CHANGE_TYPE_DONE) {
 246                 /* A change in our content has gone through.  Re-build our pieces. */
 247                 setup_pieces ();
 248                 --_suspended;
 249         } else if (type == CHANGE_TYPE_CANCELLED) {
 250                 --_suspended;
 251         }
 252
 253         Change (type, property, frequent);
 254 }
 255
 256 void
 257 Player::set_video_container_size (dcp::Size s)
 258 {
 259         Change (CHANGE_TYPE_PENDING, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 260
 261         {
 262                 boost::mutex::scoped_lock lm (_mutex);
 263
 264                 if (s == _video_container_size) {
 265                         lm.unlock ();
 266                         Change (CHANGE_TYPE_CANCELLED, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 267                         return;
 268                 }
 269
 270                 _video_container_size = s;
 271
 272                 _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 273                 _black_image->make_black ();
 274         }
 275
 276         Change (CHANGE_TYPE_DONE, PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 277 }
 278
 279 void
 280 Player::playlist_change (ChangeType type)
 281 {
 282         if (type == CHANGE_TYPE_DONE) {
 283                 setup_pieces ();
 284         }
 285         Change (type, PlayerProperty::PLAYLIST, false);
 286 }
 287
 288 void
 289 Player::film_change (ChangeType type, Film::Property p)
 290 {
 291         /* Here we should notice Film properties that affect our output, and
 292            alert listeners that our output now would be different to how it was
 293            last time we were run.
 294         */
 295
 296         if (p == Film::CONTAINER) {
 297                 Change (type, PlayerProperty::FILM_CONTAINER, false);
 298         } else if (p == Film::VIDEO_FRAME_RATE) {
 299                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 300                    so we need new pieces here.
 301                 */
 302                 if (type == CHANGE_TYPE_DONE) {
 303                         setup_pieces ();
 304                 }
 305                 Change (type, PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
 306         } else if (p == Film::AUDIO_PROCESSOR) {
 307                 if (type == CHANGE_TYPE_DONE && _film->audio_processor ()) {
 308                         boost::mutex::scoped_lock lm (_mutex);
 309                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 310                 }
 311         } else if (p == Film::AUDIO_CHANNELS) {
 312                 if (type == CHANGE_TYPE_DONE) {
 313                         boost::mutex::scoped_lock lm (_mutex);
 314                         _audio_merger.clear ();
 315                 }
 316         }
 317 }
 318
 319 shared_ptr<PlayerVideo>
 320 Player::black_player_video_frame (Eyes eyes) const
 321 {
 322         return shared_ptr<PlayerVideo> (
 323                 new PlayerVideo (
 324                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 325                         Crop (),
 326                         optional<double> (),
 327                         _video_container_size,
 328                         _video_container_size,
 329                         eyes,
 330                         PART_WHOLE,
 331                         PresetColourConversion::all().front().conversion,
 332                         boost::weak_ptr<Content>(),
 333                         boost::optional<Frame>()
 334                 )
 335         );
 336 }
 337
 338 Frame
 339 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 340 {
 341         DCPTime s = t - piece->content->position ();
 342         s = min (piece->content->length_after_trim(_film), s);
 343         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 344
 345         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 346            then convert that ContentTime to frames at the content's rate.  However this fails for
 347            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 348            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 349
 350            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 351         */
 352         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 353 }
 354
 355 DCPTime
 356 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 357 {
 358         /* See comment in dcp_to_content_video */
 359         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime(piece->content->trim_start(), piece->frc);
 360         return d + piece->content->position();
 361 }
 362
 363 Frame
 364 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 365 {
 366         DCPTime s = t - piece->content->position ();
 367         s = min (piece->content->length_after_trim(_film), s);
 368         /* See notes in dcp_to_content_video */
 369         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 370 }
 371
 372 DCPTime
 373 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 374 {
 375         /* See comment in dcp_to_content_video */
 376         return DCPTime::from_frames (f, _film->audio_frame_rate())
 377                 - DCPTime (piece->content->trim_start(), piece->frc)
 378                 + piece->content->position();
 379 }
 380
 381 ContentTime
 382 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 383 {
 384         DCPTime s = t - piece->content->position ();
 385         s = min (piece->content->length_after_trim(_film), s);
 386         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 387 }
 388
 389 DCPTime
 390 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 391 {
 392         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 393 }
 394
 395 list<shared_ptr<Font> >
 396 Player::get_subtitle_fonts ()
 397 {
 398         boost::mutex::scoped_lock lm (_mutex);
 399
 400         list<shared_ptr<Font> > fonts;
 401         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 402                 BOOST_FOREACH (shared_ptr<TextContent> j, i->content->text) {
 403                         /* XXX: things may go wrong if there are duplicate font IDs
 404                            with different font files.
 405                         */
 406                         list<shared_ptr<Font> > f = j->fonts ();
 407                         copy (f.begin(), f.end(), back_inserter (fonts));
 408                 }
 409         }
 410
 411         return fonts;
 412 }
 413
 414 /** Set this player never to produce any video data */
 415 void
 416 Player::set_ignore_video ()
 417 {
 418         boost::mutex::scoped_lock lm (_mutex);
 419         _ignore_video = true;
 420         setup_pieces_unlocked ();
 421 }
 422
 423 void
 424 Player::set_ignore_audio ()
 425 {
 426         boost::mutex::scoped_lock lm (_mutex);
 427         _ignore_audio = true;
 428         setup_pieces_unlocked ();
 429 }
 430
 431 void
 432 Player::set_ignore_text ()
 433 {
 434         boost::mutex::scoped_lock lm (_mutex);
 435         _ignore_text = true;
 436         setup_pieces_unlocked ();
 437 }
 438
 439 /** Set the player to always burn open texts into the image regardless of the content settings */
 440 void
 441 Player::set_always_burn_open_subtitles ()
 442 {
 443         boost::mutex::scoped_lock lm (_mutex);
 444         _always_burn_open_subtitles = true;
 445 }
 446
 447 /** Sets up the player to be faster, possibly at the expense of quality */
 448 void
 449 Player::set_fast ()
 450 {
 451         boost::mutex::scoped_lock lm (_mutex);
 452         _fast = true;
 453         setup_pieces_unlocked ();
 454 }
 455
 456 void
 457 Player::set_play_referenced ()
 458 {
 459         boost::mutex::scoped_lock lm (_mutex);
 460         _play_referenced = true;
 461         setup_pieces_unlocked ();
 462 }
 463
 464 static void
 465 maybe_add_asset (list<ReferencedReelAsset>& a, shared_ptr<dcp::ReelAsset> r, Frame reel_trim_start, Frame reel_trim_end, DCPTime from, int const ffr)
 466 {
 467         DCPOMATIC_ASSERT (r);
 468         r->set_entry_point (r->entry_point() + reel_trim_start);
 469         r->set_duration (r->duration() - reel_trim_start - reel_trim_end);
 470         if (r->duration() > 0) {
 471                 a.push_back (
 472                         ReferencedReelAsset(r, DCPTimePeriod(from, from + DCPTime::from_frames(r->duration(), ffr)))
 473                         );
 474         }
 475 }
 476
 477 list<ReferencedReelAsset>
 478 Player::get_reel_assets ()
 479 {
 480         /* Does not require a lock on _mutex as it's only called from DCPEncoder */
 481
 482         list<ReferencedReelAsset> a;
 483
 484         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 485                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 486                 if (!j) {
 487                         continue;
 488                 }
 489
 490                 scoped_ptr<DCPDecoder> decoder;
 491                 try {
 492                         decoder.reset (new DCPDecoder (_film, j, false));
 493                 } catch (...) {
 494                         return a;
 495                 }
 496
 497                 DCPOMATIC_ASSERT (j->video_frame_rate ());
 498                 double const cfr = j->video_frame_rate().get();
 499                 Frame const trim_start = j->trim_start().frames_round (cfr);
 500                 Frame const trim_end = j->trim_end().frames_round (cfr);
 501                 int const ffr = _film->video_frame_rate ();
 502
 503                 /* position in the asset from the start */
 504                 int64_t offset_from_start = 0;
 505                 /* position in the asset from the end */
 506                 int64_t offset_from_end = 0;
 507                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 508                         /* Assume that main picture duration is the length of the reel */
 509                         offset_from_end += k->main_picture()->duration();
 510                 }
 511
 512                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 513
 514                         /* Assume that main picture duration is the length of the reel */
 515                         int64_t const reel_duration = k->main_picture()->duration();
 516
 517                         /* See doc/design/trim_reels.svg */
 518                         Frame const reel_trim_start = min(reel_duration, max(int64_t(0), trim_start - offset_from_start));
 519                         Frame const reel_trim_end =   min(reel_duration, max(int64_t(0), reel_duration - (offset_from_end - trim_end)));
 520
 521                         DCPTime const from = i->position() + DCPTime::from_frames (offset_from_start, _film->video_frame_rate());
 522                         if (j->reference_video ()) {
 523                                 maybe_add_asset (a, k->main_picture(), reel_trim_start, reel_trim_end, from, ffr);
 524                         }
 525
 526                         if (j->reference_audio ()) {
 527                                 maybe_add_asset (a, k->main_sound(), reel_trim_start, reel_trim_end, from, ffr);
 528                         }
 529
 530                         if (j->reference_text (TEXT_OPEN_SUBTITLE)) {
 531                                 maybe_add_asset (a, k->main_subtitle(), reel_trim_start, reel_trim_end, from, ffr);
 532                         }
 533
 534                         if (j->reference_text (TEXT_CLOSED_CAPTION)) {
 535                                 BOOST_FOREACH (shared_ptr<dcp::ReelClosedCaptionAsset> l, k->closed_captions()) {
 536                                         maybe_add_asset (a, l, reel_trim_start, reel_trim_end, from, ffr);
 537                                 }
 538                         }
 539
 540                         offset_from_start += reel_duration;
 541                         offset_from_end -= reel_duration;
 542                 }
 543         }
 544
 545         return a;
 546 }
 547
 548 bool
 549 Player::pass ()
 550 {
 551         boost::mutex::scoped_lock lm (_mutex);
 552
 553         if (_suspended) {
 554                 /* We can't pass in this state */
 555                 return false;
 556         }
 557
 558         if (_playlist->length(_film) == DCPTime()) {
 559                 /* Special case of an empty Film; just give one black frame */
 560                 emit_video (black_player_video_frame(EYES_BOTH), DCPTime());
 561                 return true;
 562         }
 563
 564         /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
 565
 566         shared_ptr<Piece> earliest_content;
 567         optional<DCPTime> earliest_time;
 568
 569         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 570                 if (i->done) {
 571                         continue;
 572                 }
 573
 574                 DCPTime const t = content_time_to_dcp (i, max(i->decoder->position(), i->content->trim_start()));
 575                 if (t > i->content->end(_film)) {
 576                         i->done = true;
 577                 } else {
 578
 579                         /* Given two choices at the same time, pick the one with texts so we see it before
 580                            the video.
 581                         */
 582                         if (!earliest_time || t < *earliest_time || (t == *earliest_time && !i->decoder->text.empty())) {
 583                                 earliest_time = t;
 584                                 earliest_content = i;
 585                         }
 586                 }
 587         }
 588
 589         bool done = false;
 590
 591         enum {
 592                 NONE,
 593                 CONTENT,
 594                 BLACK,
 595                 SILENT
 596         } which = NONE;
 597
 598         if (earliest_content) {
 599                 which = CONTENT;
 600         }
 601
 602         if (!_black.done() && (!earliest_time || _black.position() < *earliest_time)) {
 603                 earliest_time = _black.position ();
 604                 which = BLACK;
 605         }
 606
 607         if (!_silent.done() && (!earliest_time || _silent.position() < *earliest_time)) {
 608                 earliest_time = _silent.position ();
 609                 which = SILENT;
 610         }
 611
 612         switch (which) {
 613         case CONTENT:
 614         {
 615                 earliest_content->done = earliest_content->decoder->pass ();
 616                 shared_ptr<DCPContent> dcp = dynamic_pointer_cast<DCPContent>(earliest_content->content);
 617                 if (dcp && !_play_referenced && dcp->reference_audio()) {
 618                         /* We are skipping some referenced DCP audio content, so we need to update _last_audio_time
 619                            to `hide' the fact that no audio was emitted during the referenced DCP (though
 620                            we need to behave as though it was).
 621                         */
 622                         _last_audio_time = dcp->end (_film);
 623                 }
 624                 break;
 625         }
 626         case BLACK:
 627                 emit_video (black_player_video_frame(EYES_BOTH), _black.position());
 628                 _black.set_position (_black.position() + one_video_frame());
 629                 break;
 630         case SILENT:
 631         {
 632                 DCPTimePeriod period (_silent.period_at_position());
 633                 if (_last_audio_time) {
 634                         /* Sometimes the thing that happened last finishes fractionally before
 635                            or after this silence.  Bodge the start time of the silence to fix it.
 636                            I think this is nothing to worry about since we will just add or
 637                            remove a little silence at the end of some content.
 638                         */
 639                         int64_t const error = labs(period.from.get() - _last_audio_time->get());
 640                         /* Let's not worry about less than a frame at 24fps */
 641                         int64_t const too_much_error = DCPTime::from_frames(1, 24).get();
 642                         if (error >= too_much_error) {
 643                                 _film->log()->log(String::compose("Silence starting before or after last audio by %1", error), LogEntry::TYPE_ERROR);
 644                         }
 645                         DCPOMATIC_ASSERT (error < too_much_error);
 646                         period.from = *_last_audio_time;
 647                 }
 648                 if (period.duration() > one_video_frame()) {
 649                         period.to = period.from + one_video_frame();
 650                 }
 651                 fill_audio (period);
 652                 _silent.set_position (period.to);
 653                 break;
 654         }
 655         case NONE:
 656                 done = true;
 657                 break;
 658         }
 659
 660         /* Emit any audio that is ready */
 661
 662         /* Work out the time before which the audio is definitely all here.  This is the earliest last_push_end of one
 663            of our streams, or the position of the _silent.
 664         */
 665         DCPTime pull_to = _film->length ();
 666         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 667                 if (!i->second.piece->done && i->second.last_push_end < pull_to) {
 668                         pull_to = i->second.last_push_end;
 669                 }
 670         }
 671         if (!_silent.done() && _silent.position() < pull_to) {
 672                 pull_to = _silent.position();
 673         }
 674
 675         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 676         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 677                 if (_last_audio_time && i->second < *_last_audio_time) {
 678                         /* This new data comes before the last we emitted (or the last seek); discard it */
 679                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 680                         if (!cut.first) {
 681                                 continue;
 682                         }
 683                         *i = cut;
 684                 } else if (_last_audio_time && i->second > *_last_audio_time) {
 685                         /* There's a gap between this data and the last we emitted; fill with silence */
 686                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 687                 }
 688
 689                 emit_audio (i->first, i->second);
 690         }
 691
 692         if (done) {
 693                 _shuffler->flush ();
 694                 for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 695                         do_emit_video(i->first, i->second);
 696                 }
 697         }
 698
 699         return done;
 700 }
 701
 702 /** @return Open subtitles for the frame at the given time, converted to images */
 703 optional<PositionImage>
 704 Player::open_subtitles_for_frame (DCPTime time) const
 705 {
 706         list<PositionImage> captions;
 707         int const vfr = _film->video_frame_rate();
 708
 709         BOOST_FOREACH (
 710                 PlayerText j,
 711                 _active_texts[TEXT_OPEN_SUBTITLE].get_burnt(DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), _always_burn_open_subtitles)
 712                 ) {
 713
 714                 /* Bitmap subtitles */
 715                 BOOST_FOREACH (BitmapText i, j.bitmap) {
 716                         if (!i.image) {
 717                                 continue;
 718                         }
 719
 720                         /* i.image will already have been scaled to fit _video_container_size */
 721                         dcp::Size scaled_size (i.rectangle.width * _video_container_size.width, i.rectangle.height * _video_container_size.height);
 722
 723                         captions.push_back (
 724                                 PositionImage (
 725                                         i.image,
 726                                         Position<int> (
 727                                                 lrint (_video_container_size.width * i.rectangle.x),
 728                                                 lrint (_video_container_size.height * i.rectangle.y)
 729                                                 )
 730                                         )
 731                                 );
 732                 }
 733
 734                 /* String subtitles (rendered to an image) */
 735                 if (!j.string.empty ()) {
 736                         list<PositionImage> s = render_text (j.string, j.fonts, _video_container_size, time, vfr);
 737                         copy (s.begin(), s.end(), back_inserter (captions));
 738                 }
 739         }
 740
 741         if (captions.empty ()) {
 742                 return optional<PositionImage> ();
 743         }
 744
 745         return merge (captions);
 746 }
 747
 748 void
 749 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 750 {
 751         shared_ptr<Piece> piece = wp.lock ();
 752         if (!piece) {
 753                 return;
 754         }
 755
 756         FrameRateChange frc (_film, piece->content);
 757         if (frc.skip && (video.frame % 2) == 1) {
 758                 return;
 759         }
 760
 761         /* Time of the first frame we will emit */
 762         DCPTime const time = content_video_to_dcp (piece, video.frame);
 763
 764         /* Discard if it's before the content's period or the last accurate seek.  We can't discard
 765            if it's after the content's period here as in that case we still need to fill any gap between
 766            `now' and the end of the content's period.
 767         */
 768         if (time < piece->content->position() || (_last_video_time && time < *_last_video_time)) {
 769                 return;
 770         }
 771
 772         /* Fill gaps that we discover now that we have some video which needs to be emitted.
 773            This is where we need to fill to.
 774         */
 775         DCPTime fill_to = min (time, piece->content->end(_film));
 776
 777         if (_last_video_time) {
 778                 DCPTime fill_from = max (*_last_video_time, piece->content->position());
 779
 780                 /* Fill if we have more than half a frame to do */
 781                 if ((fill_to - fill_from) > one_video_frame() / 2) {
 782                         LastVideoMap::const_iterator last = _last_video.find (wp);
 783                         if (_film->three_d()) {
 784                                 Eyes fill_to_eyes = video.eyes;
 785                                 if (fill_to_eyes == EYES_BOTH) {
 786                                         fill_to_eyes = EYES_LEFT;
 787                                 }
 788                                 if (fill_to == piece->content->end(_film)) {
 789                                         /* Don't fill after the end of the content */
 790                                         fill_to_eyes = EYES_LEFT;
 791                                 }
 792                                 DCPTime j = fill_from;
 793                                 Eyes eyes = _last_video_eyes.get_value_or(EYES_LEFT);
 794                                 if (eyes == EYES_BOTH) {
 795                                         eyes = EYES_LEFT;
 796                                 }
 797                                 while (j < fill_to || eyes != fill_to_eyes) {
 798                                         if (last != _last_video.end()) {
 799                                                 shared_ptr<PlayerVideo> copy = last->second->shallow_copy();
 800                                                 copy->set_eyes (eyes);
 801                                                 emit_video (copy, j);
 802                                         } else {
 803                                                 emit_video (black_player_video_frame(eyes), j);
 804                                         }
 805                                         if (eyes == EYES_RIGHT) {
 806                                                 j += one_video_frame();
 807                                         }
 808                                         eyes = increment_eyes (eyes);
 809                                 }
 810                         } else {
 811                                 for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
 812                                         if (last != _last_video.end()) {
 813                                                 emit_video (last->second, j);
 814                                         } else {
 815                                                 emit_video (black_player_video_frame(EYES_BOTH), j);
 816                                         }
 817                                 }
 818                         }
 819                 }
 820         }
 821
 822         _last_video[wp].reset (
 823                 new PlayerVideo (
 824                         video.image,
 825                         piece->content->video->crop (),
 826                         piece->content->video->fade (_film, video.frame),
 827                         piece->content->video->scale().size (
 828                                 piece->content->video, _video_container_size, _film->frame_size ()
 829                                 ),
 830                         _video_container_size,
 831                         video.eyes,
 832                         video.part,
 833                         piece->content->video->colour_conversion(),
 834                         piece->content,
 835                         video.frame
 836                         )
 837                 );
 838
 839         DCPTime t = time;
 840         for (int i = 0; i < frc.repeat; ++i) {
 841                 if (t < piece->content->end(_film)) {
 842                         emit_video (_last_video[wp], t);
 843                 }
 844                 t += one_video_frame ();
 845         }
 846 }
 847
 848 void
 849 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 850 {
 851         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 852
 853         shared_ptr<Piece> piece = wp.lock ();
 854         if (!piece) {
 855                 return;
 856         }
 857
 858         shared_ptr<AudioContent> content = piece->content->audio;
 859         DCPOMATIC_ASSERT (content);
 860
 861         /* Compute time in the DCP */
 862         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame);
 863         /* And the end of this block in the DCP */
 864         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), content->resampled_frame_rate(_film));
 865
 866         /* Remove anything that comes before the start or after the end of the content */
 867         if (time < piece->content->position()) {
 868                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 869                 if (!cut.first) {
 870                         /* This audio is entirely discarded */
 871                         return;
 872                 }
 873                 content_audio.audio = cut.first;
 874                 time = cut.second;
 875         } else if (time > piece->content->end(_film)) {
 876                 /* Discard it all */
 877                 return;
 878         } else if (end > piece->content->end(_film)) {
 879                 Frame const remaining_frames = DCPTime(piece->content->end(_film) - time).frames_round(_film->audio_frame_rate());
 880                 if (remaining_frames == 0) {
 881                         return;
 882                 }
 883                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 884                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 885                 content_audio.audio = cut;
 886         }
 887
 888         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 889
 890         /* Gain */
 891
 892         if (content->gain() != 0) {
 893                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 894                 gain->apply_gain (content->gain ());
 895                 content_audio.audio = gain;
 896         }
 897
 898         /* Remap */
 899
 900         content_audio.audio = remap (content_audio.audio, _film->audio_channels(), stream->mapping());
 901
 902         /* Process */
 903
 904         if (_audio_processor) {
 905                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 906         }
 907
 908         /* Push */
 909
 910         _audio_merger.push (content_audio.audio, time);
 911         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 912         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 913 }
 914
 915 void
 916 Player::bitmap_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentBitmapText subtitle)
 917 {
 918         shared_ptr<Piece> piece = wp.lock ();
 919         shared_ptr<const TextContent> text = wc.lock ();
 920         if (!piece || !text) {
 921                 return;
 922         }
 923
 924         /* Apply content's subtitle offsets */
 925         subtitle.sub.rectangle.x += text->x_offset ();
 926         subtitle.sub.rectangle.y += text->y_offset ();
 927
 928         /* Apply a corrective translation to keep the subtitle centred after the scale that is coming up */
 929         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * ((text->x_scale() - 1) / 2);
 930         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * ((text->y_scale() - 1) / 2);
 931
 932         /* Apply content's subtitle scale */
 933         subtitle.sub.rectangle.width *= text->x_scale ();
 934         subtitle.sub.rectangle.height *= text->y_scale ();
 935
 936         PlayerText ps;
 937         shared_ptr<Image> image = subtitle.sub.image;
 938
 939         /* We will scale the subtitle up to fit _video_container_size */
 940         int const width = subtitle.sub.rectangle.width * _video_container_size.width;
 941         int const height = subtitle.sub.rectangle.height * _video_container_size.height;
 942         if (width == 0 || height == 0) {
 943                 return;
 944         }
 945
 946         dcp::Size scaled_size (width, height);
 947         ps.bitmap.push_back (BitmapText(image->scale(scaled_size, dcp::YUV_TO_RGB_REC601, image->pixel_format(), true, _fast), subtitle.sub.rectangle));
 948         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 949
 950         _active_texts[text->type()].add_from (wc, ps, from);
 951 }
 952
 953 void
 954 Player::plain_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentStringText subtitle)
 955 {
 956         shared_ptr<Piece> piece = wp.lock ();
 957         shared_ptr<const TextContent> text = wc.lock ();
 958         if (!piece || !text) {
 959                 return;
 960         }
 961
 962         PlayerText ps;
 963         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 964
 965         if (from > piece->content->end(_film)) {
 966                 return;
 967         }
 968
 969         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 970                 s.set_h_position (s.h_position() + text->x_offset ());
 971                 s.set_v_position (s.v_position() + text->y_offset ());
 972                 float const xs = text->x_scale();
 973                 float const ys = text->y_scale();
 974                 float size = s.size();
 975
 976                 /* Adjust size to express the common part of the scaling;
 977                    e.g. if xs = ys = 0.5 we scale size by 2.
 978                 */
 979                 if (xs > 1e-5 && ys > 1e-5) {
 980                         size *= 1 / min (1 / xs, 1 / ys);
 981                 }
 982                 s.set_size (size);
 983
 984                 /* Then express aspect ratio changes */
 985                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 986                         s.set_aspect_adjust (xs / ys);
 987                 }
 988
 989                 s.set_in (dcp::Time(from.seconds(), 1000));
 990                 ps.string.push_back (StringText (s, text->outline_width()));
 991                 ps.add_fonts (text->fonts ());
 992         }
 993
 994         _active_texts[text->type()].add_from (wc, ps, from);
 995 }
 996
 997 void
 998 Player::subtitle_stop (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentTime to)
 999 {
1000         shared_ptr<const TextContent> text = wc.lock ();
1001         if (!text) {
1002                 return;
1003         }
1004
1005         if (!_active_texts[text->type()].have(wc)) {
1006                 return;
1007         }
1008
1009         shared_ptr<Piece> piece = wp.lock ();
1010         if (!piece) {
1011                 return;
1012         }
1013
1014         DCPTime const dcp_to = content_time_to_dcp (piece, to);
1015
1016         if (dcp_to > piece->content->end(_film)) {
1017                 return;
1018         }
1019
1020         pair<PlayerText, DCPTime> from = _active_texts[text->type()].add_to (wc, dcp_to);
1021
1022         bool const always = (text->type() == TEXT_OPEN_SUBTITLE && _always_burn_open_subtitles);
1023         if (text->use() && !always && !text->burn()) {
1024                 Text (from.first, text->type(), text->dcp_track().get_value_or(DCPTextTrack()), DCPTimePeriod (from.second, dcp_to));
1025         }
1026 }
1027
1028 void
1029 Player::seek (DCPTime time, bool accurate)
1030 {
1031         boost::mutex::scoped_lock lm (_mutex);
1032
1033         if (_suspended) {
1034                 /* We can't seek in this state */
1035                 return;
1036         }
1037
1038         if (_shuffler) {
1039                 _shuffler->clear ();
1040         }
1041
1042         _delay.clear ();
1043
1044         if (_audio_processor) {
1045                 _audio_processor->flush ();
1046         }
1047
1048         _audio_merger.clear ();
1049         for (int i = 0; i < TEXT_COUNT; ++i) {
1050                 _active_texts[i].clear ();
1051         }
1052
1053         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1054                 if (time < i->content->position()) {
1055                         /* Before; seek to the start of the content */
1056                         i->decoder->seek (dcp_to_content_time (i, i->content->position()), accurate);
1057                         i->done = false;
1058                 } else if (i->content->position() <= time && time < i->content->end(_film)) {
1059                         /* During; seek to position */
1060                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
1061                         i->done = false;
1062                 } else {
1063                         /* After; this piece is done */
1064                         i->done = true;
1065                 }
1066         }
1067
1068         if (accurate) {
1069                 _last_video_time = time;
1070                 _last_video_eyes = EYES_LEFT;
1071                 _last_audio_time = time;
1072         } else {
1073                 _last_video_time = optional<DCPTime>();
1074                 _last_video_eyes = optional<Eyes>();
1075                 _last_audio_time = optional<DCPTime>();
1076         }
1077
1078         _black.set_position (time);
1079         _silent.set_position (time);
1080
1081         _last_video.clear ();
1082 }
1083
1084 void
1085 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1086 {
1087         /* We need a delay to give a little wiggle room to ensure that relevent subtitles arrive at the
1088            player before the video that requires them.
1089         */
1090         _delay.push_back (make_pair (pv, time));
1091
1092         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1093                 _last_video_time = time + one_video_frame();
1094         }
1095         _last_video_eyes = increment_eyes (pv->eyes());
1096
1097         if (_delay.size() < 3) {
1098                 return;
1099         }
1100
1101         pair<shared_ptr<PlayerVideo>, DCPTime> to_do = _delay.front();
1102         _delay.pop_front();
1103         do_emit_video (to_do.first, to_do.second);
1104 }
1105
1106 void
1107 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1108 {
1109         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1110                 for (int i = 0; i < TEXT_COUNT; ++i) {
1111                         _active_texts[i].clear_before (time);
1112                 }
1113         }
1114
1115         optional<PositionImage> subtitles = open_subtitles_for_frame (time);
1116         if (subtitles) {
1117                 pv->set_text (subtitles.get ());
1118         }
1119
1120         Video (pv, time);
1121 }
1122
1123 void
1124 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1125 {
1126         /* Log if the assert below is about to fail */
1127         if (_last_audio_time && labs(time.get() - _last_audio_time->get()) > 1) {
1128                 _film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_last_audio_time)), LogEntry::TYPE_WARNING);
1129         }
1130
1131         /* This audio must follow on from the previous, allowing for half a sample (at 48kHz) leeway */
1132         DCPOMATIC_ASSERT (!_last_audio_time || labs(time.get() - _last_audio_time->get()) < 2);
1133         Audio (data, time, _film->audio_frame_rate());
1134         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate());
1135 }
1136
1137 void
1138 Player::fill_audio (DCPTimePeriod period)
1139 {
1140         if (period.from == period.to) {
1141                 return;
1142         }
1143
1144         DCPOMATIC_ASSERT (period.from < period.to);
1145
1146         DCPTime t = period.from;
1147         while (t < period.to) {
1148                 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1149                 Frame const samples = block.frames_round(_film->audio_frame_rate());
1150                 if (samples) {
1151                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1152                         silence->make_silent ();
1153                         emit_audio (silence, t);
1154                 }
1155                 t += block;
1156         }
1157 }
1158
1159 DCPTime
1160 Player::one_video_frame () const
1161 {
1162         return DCPTime::from_frames (1, _film->video_frame_rate ());
1163 }
1164
1165 pair<shared_ptr<AudioBuffers>, DCPTime>
1166 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1167 {
1168         DCPTime const discard_time = discard_to - time;
1169         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1170         Frame remaining_frames = audio->frames() - discard_frames;
1171         if (remaining_frames <= 0) {
1172                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1173         }
1174         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1175         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1176         return make_pair(cut, time + discard_time);
1177 }
1178
1179 void
1180 Player::set_dcp_decode_reduction (optional<int> reduction)
1181 {
1182         Change (CHANGE_TYPE_PENDING, PlayerProperty::DCP_DECODE_REDUCTION, false);
1183
1184         {
1185                 boost::mutex::scoped_lock lm (_mutex);
1186
1187                 if (reduction == _dcp_decode_reduction) {
1188                         lm.unlock ();
1189                         Change (CHANGE_TYPE_CANCELLED, PlayerProperty::DCP_DECODE_REDUCTION, false);
1190                         return;
1191                 }
1192
1193                 _dcp_decode_reduction = reduction;
1194                 setup_pieces_unlocked ();
1195         }
1196
1197         Change (CHANGE_TYPE_DONE, PlayerProperty::DCP_DECODE_REDUCTION, false);
1198 }
1199
1200 optional<DCPTime>
1201 Player::content_time_to_dcp (shared_ptr<Content> content, ContentTime t)
1202 {
1203         boost::mutex::scoped_lock lm (_mutex);
1204
1205         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1206                 if (i->content == content) {
1207                         return content_time_to_dcp (i, t);
1208                 }
1209         }
1210
1211         /* We couldn't find this content; perhaps things are being changed over */
1212         return optional<DCPTime>();
1213 }