src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2018 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_text.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "text_content.h"
  44 #include "text_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "dcp_decoder.h"
  48 #include "image_decoder.h"
  49 #include "compose.hpp"
  50 #include "shuffler.h"
  51 #include <dcp/reel.h>
  52 #include <dcp/reel_sound_asset.h>
  53 #include <dcp/reel_subtitle_asset.h>
  54 #include <dcp/reel_picture_asset.h>
  55 #include <dcp/reel_closed_caption_asset.h>
  56 #include <boost/foreach.hpp>
  57 #include <stdint.h>
  58 #include <algorithm>
  59 #include <iostream>
  60
  61 #include "i18n.h"
  62
  63 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
  64
  65 using std::list;
  66 using std::cout;
  67 using std::min;
  68 using std::max;
  69 using std::min;
  70 using std::vector;
  71 using std::pair;
  72 using std::map;
  73 using std::make_pair;
  74 using std::copy;
  75 using boost::shared_ptr;
  76 using boost::weak_ptr;
  77 using boost::dynamic_pointer_cast;
  78 using boost::optional;
  79 using boost::scoped_ptr;
  80
  81 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
  82 int const PlayerProperty::PLAYLIST = 701;
  83 int const PlayerProperty::FILM_CONTAINER = 702;
  84 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
  85 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
  86
  87 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  88         : _film (film)
  89         , _playlist (playlist)
  90         , _have_valid_pieces (false)
  91         , _ignore_video (false)
  92         , _ignore_audio (false)
  93         , _ignore_text (false)
  94         , _always_burn_open_subtitles (false)
  95         , _fast (false)
  96         , _play_referenced (false)
  97         , _audio_merger (_film->audio_frame_rate())
  98         , _shuffler (0)
  99 {
 100         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
 101         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 102         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
 103         set_video_container_size (_film->frame_size ());
 104
 105         film_changed (Film::AUDIO_PROCESSOR);
 106
 107         seek (DCPTime (), true);
 108 }
 109
 110 Player::~Player ()
 111 {
 112         delete _shuffler;
 113 }
 114
 115 void
 116 Player::setup_pieces ()
 117 {
 118         _pieces.clear ();
 119
 120         delete _shuffler;
 121         _shuffler = new Shuffler();
 122         _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
 123
 124         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 125
 126                 if (!i->paths_valid ()) {
 127                         continue;
 128                 }
 129
 130                 if (_ignore_video && _ignore_audio && i->text.empty()) {
 131                         /* We're only interested in text and this content has none */
 132                         continue;
 133                 }
 134
 135                 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log(), _fast);
 136                 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
 137
 138                 if (!decoder) {
 139                         /* Not something that we can decode; e.g. Atmos content */
 140                         continue;
 141                 }
 142
 143                 if (decoder->video && _ignore_video) {
 144                         decoder->video->set_ignore (true);
 145                 }
 146
 147                 if (decoder->audio && _ignore_audio) {
 148                         decoder->audio->set_ignore (true);
 149                 }
 150
 151                 if (_ignore_text) {
 152                         BOOST_FOREACH (shared_ptr<TextDecoder> i, decoder->text) {
 153                                 i->set_ignore (true);
 154                         }
 155                 }
 156
 157                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 158                 if (dcp) {
 159                         dcp->set_decode_referenced (_play_referenced);
 160                         if (_play_referenced) {
 161                                 dcp->set_forced_reduction (_dcp_decode_reduction);
 162                         }
 163                 }
 164
 165                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 166                 _pieces.push_back (piece);
 167
 168                 if (decoder->video) {
 169                         if (i->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT || i->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) {
 170                                 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
 171                                 decoder->video->Data.connect (bind (&Shuffler::video, _shuffler, weak_ptr<Piece>(piece), _1));
 172                         } else {
 173                                 decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece>(piece), _1));
 174                         }
 175                 }
 176
 177                 if (decoder->audio) {
 178                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 179                 }
 180
 181                 list<shared_ptr<TextDecoder> >::const_iterator j = decoder->text.begin();
 182
 183                 while (j != decoder->text.end()) {
 184                         (*j)->BitmapStart.connect (
 185                                 bind(&Player::bitmap_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 186                                 );
 187                         (*j)->PlainStart.connect (
 188                                 bind(&Player::plain_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 189                                 );
 190                         (*j)->Stop.connect (
 191                                 bind(&Player::subtitle_stop, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1, _2)
 192                                 );
 193
 194                         ++j;
 195                 }
 196         }
 197
 198         _stream_states.clear ();
 199         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 200                 if (i->content->audio) {
 201                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 202                                 _stream_states[j] = StreamState (i, i->content->position ());
 203                         }
 204                 }
 205         }
 206
 207         _black = Empty (_film->content(), _film->length(), bind(&Content::video, _1));
 208         _silent = Empty (_film->content(), _film->length(), bind(&Content::audio, _1));
 209
 210         _last_video_time = DCPTime ();
 211         _last_video_eyes = EYES_BOTH;
 212         _last_audio_time = DCPTime ();
 213         _have_valid_pieces = true;
 214 }
 215
 216 void
 217 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
 218 {
 219         shared_ptr<Content> c = w.lock ();
 220         if (!c) {
 221                 return;
 222         }
 223
 224         if (
 225                 property == ContentProperty::POSITION ||
 226                 property == ContentProperty::LENGTH ||
 227                 property == ContentProperty::TRIM_START ||
 228                 property == ContentProperty::TRIM_END ||
 229                 property == ContentProperty::PATH ||
 230                 property == VideoContentProperty::FRAME_TYPE ||
 231                 property == VideoContentProperty::COLOUR_CONVERSION ||
 232                 property == AudioContentProperty::STREAMS ||
 233                 property == DCPContentProperty::NEEDS_ASSETS ||
 234                 property == DCPContentProperty::NEEDS_KDM ||
 235                 property == TextContentProperty::COLOUR ||
 236                 property == TextContentProperty::EFFECT ||
 237                 property == TextContentProperty::EFFECT_COLOUR ||
 238                 property == FFmpegContentProperty::SUBTITLE_STREAM ||
 239                 property == FFmpegContentProperty::FILTERS
 240                 ) {
 241
 242                 {
 243                         boost::mutex::scoped_lock lm (_mutex);
 244                         _have_valid_pieces = false;
 245                 }
 246
 247                 Changed (property, frequent);
 248
 249         } else if (
 250                 property == TextContentProperty::LINE_SPACING ||
 251                 property == TextContentProperty::OUTLINE_WIDTH ||
 252                 property == TextContentProperty::Y_SCALE ||
 253                 property == TextContentProperty::FADE_IN ||
 254                 property == TextContentProperty::FADE_OUT ||
 255                 property == ContentProperty::VIDEO_FRAME_RATE ||
 256                 property == TextContentProperty::USE ||
 257                 property == TextContentProperty::X_OFFSET ||
 258                 property == TextContentProperty::Y_OFFSET ||
 259                 property == TextContentProperty::X_SCALE ||
 260                 property == TextContentProperty::FONTS ||
 261                 property == TextContentProperty::TYPE ||
 262                 property == VideoContentProperty::CROP ||
 263                 property == VideoContentProperty::SCALE ||
 264                 property == VideoContentProperty::FADE_IN ||
 265                 property == VideoContentProperty::FADE_OUT
 266                 ) {
 267
 268                 Changed (property, frequent);
 269         }
 270 }
 271
 272 void
 273 Player::set_video_container_size (dcp::Size s)
 274 {
 275         {
 276                 boost::mutex::scoped_lock lm (_mutex);
 277
 278                 if (s == _video_container_size) {
 279                         return;
 280                 }
 281
 282                 _video_container_size = s;
 283
 284                 _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 285                 _black_image->make_black ();
 286         }
 287
 288         Changed (PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 289 }
 290
 291 void
 292 Player::playlist_changed ()
 293 {
 294         {
 295                 boost::mutex::scoped_lock lm (_mutex);
 296                 _have_valid_pieces = false;
 297         }
 298
 299         Changed (PlayerProperty::PLAYLIST, false);
 300 }
 301
 302 void
 303 Player::film_changed (Film::Property p)
 304 {
 305         /* Here we should notice Film properties that affect our output, and
 306            alert listeners that our output now would be different to how it was
 307            last time we were run.
 308         */
 309
 310         if (p == Film::CONTAINER) {
 311                 Changed (PlayerProperty::FILM_CONTAINER, false);
 312         } else if (p == Film::VIDEO_FRAME_RATE) {
 313                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 314                    so we need new pieces here.
 315                 */
 316                 {
 317                         boost::mutex::scoped_lock lm (_mutex);
 318                         _have_valid_pieces = false;
 319                 }
 320                 Changed (PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
 321         } else if (p == Film::AUDIO_PROCESSOR) {
 322                 if (_film->audio_processor ()) {
 323                         boost::mutex::scoped_lock lm (_mutex);
 324                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 325                 }
 326         } else if (p == Film::AUDIO_CHANNELS) {
 327                 boost::mutex::scoped_lock lm (_mutex);
 328                 _audio_merger.clear ();
 329         }
 330 }
 331
 332 list<PositionImage>
 333 Player::transform_bitmap_texts (list<BitmapText> subs) const
 334 {
 335         list<PositionImage> all;
 336
 337         for (list<BitmapText>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 338                 if (!i->image) {
 339                         continue;
 340                 }
 341
 342                 /* We will scale the subtitle up to fit _video_container_size */
 343                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
 344
 345                 all.push_back (
 346                         PositionImage (
 347                                 i->image->scale (
 348                                         scaled_size,
 349                                         dcp::YUV_TO_RGB_REC601,
 350                                         i->image->pixel_format (),
 351                                         true,
 352                                         _fast
 353                                         ),
 354                                 Position<int> (
 355                                         lrint (_video_container_size.width * i->rectangle.x),
 356                                         lrint (_video_container_size.height * i->rectangle.y)
 357                                         )
 358                                 )
 359                         );
 360         }
 361
 362         return all;
 363 }
 364
 365 shared_ptr<PlayerVideo>
 366 Player::black_player_video_frame (Eyes eyes) const
 367 {
 368         return shared_ptr<PlayerVideo> (
 369                 new PlayerVideo (
 370                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 371                         Crop (),
 372                         optional<double> (),
 373                         _video_container_size,
 374                         _video_container_size,
 375                         eyes,
 376                         PART_WHOLE,
 377                         PresetColourConversion::all().front().conversion,
 378                         boost::weak_ptr<Content>(),
 379                         boost::optional<Frame>()
 380                 )
 381         );
 382 }
 383
 384 Frame
 385 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 386 {
 387         DCPTime s = t - piece->content->position ();
 388         s = min (piece->content->length_after_trim(), s);
 389         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 390
 391         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 392            then convert that ContentTime to frames at the content's rate.  However this fails for
 393            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 394            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 395
 396            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 397         */
 398         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 399 }
 400
 401 DCPTime
 402 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 403 {
 404         /* See comment in dcp_to_content_video */
 405         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime(piece->content->trim_start(), piece->frc);
 406         return d + piece->content->position();
 407 }
 408
 409 Frame
 410 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 411 {
 412         DCPTime s = t - piece->content->position ();
 413         s = min (piece->content->length_after_trim(), s);
 414         /* See notes in dcp_to_content_video */
 415         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 416 }
 417
 418 DCPTime
 419 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 420 {
 421         /* See comment in dcp_to_content_video */
 422         return DCPTime::from_frames (f, _film->audio_frame_rate())
 423                 - DCPTime (piece->content->trim_start(), piece->frc)
 424                 + piece->content->position();
 425 }
 426
 427 ContentTime
 428 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 429 {
 430         DCPTime s = t - piece->content->position ();
 431         s = min (piece->content->length_after_trim(), s);
 432         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 433 }
 434
 435 DCPTime
 436 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 437 {
 438         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 439 }
 440
 441 list<shared_ptr<Font> >
 442 Player::get_subtitle_fonts ()
 443 {
 444         /* Does not require a lock on _mutex as it's only called from DCPEncoder */
 445
 446         if (!_have_valid_pieces) {
 447                 setup_pieces ();
 448         }
 449
 450         list<shared_ptr<Font> > fonts;
 451         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 452                 BOOST_FOREACH (shared_ptr<TextContent> j, i->content->text) {
 453                         /* XXX: things may go wrong if there are duplicate font IDs
 454                            with different font files.
 455                         */
 456                         list<shared_ptr<Font> > f = j->fonts ();
 457                         copy (f.begin(), f.end(), back_inserter (fonts));
 458                 }
 459         }
 460
 461         return fonts;
 462 }
 463
 464 /** Set this player never to produce any video data */
 465 void
 466 Player::set_ignore_video ()
 467 {
 468         boost::mutex::scoped_lock lm (_mutex);
 469         _ignore_video = true;
 470         _have_valid_pieces = false;
 471 }
 472
 473 void
 474 Player::set_ignore_audio ()
 475 {
 476         boost::mutex::scoped_lock lm (_mutex);
 477         _ignore_audio = true;
 478         _have_valid_pieces = false;
 479 }
 480
 481 void
 482 Player::set_ignore_text ()
 483 {
 484         boost::mutex::scoped_lock lm (_mutex);
 485         _ignore_text = true;
 486 }
 487
 488 /** Set the player to always burn open texts into the image regardless of the content settings */
 489 void
 490 Player::set_always_burn_open_subtitles ()
 491 {
 492         boost::mutex::scoped_lock lm (_mutex);
 493         _always_burn_open_subtitles = true;
 494 }
 495
 496 /** Sets up the player to be faster, possibly at the expense of quality */
 497 void
 498 Player::set_fast ()
 499 {
 500         boost::mutex::scoped_lock lm (_mutex);
 501         _fast = true;
 502         _have_valid_pieces = false;
 503 }
 504
 505 void
 506 Player::set_play_referenced ()
 507 {
 508         boost::mutex::scoped_lock lm (_mutex);
 509         _play_referenced = true;
 510         _have_valid_pieces = false;
 511 }
 512
 513 list<ReferencedReelAsset>
 514 Player::get_reel_assets ()
 515 {
 516         /* Does not require a lock on _mutex as it's only called from DCPEncoder */
 517
 518         list<ReferencedReelAsset> a;
 519
 520         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 521                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 522                 if (!j) {
 523                         continue;
 524                 }
 525
 526                 scoped_ptr<DCPDecoder> decoder;
 527                 try {
 528                         decoder.reset (new DCPDecoder (j, _film->log(), false));
 529                 } catch (...) {
 530                         return a;
 531                 }
 532
 533                 int64_t offset = 0;
 534                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 535
 536                         DCPOMATIC_ASSERT (j->video_frame_rate ());
 537                         double const cfr = j->video_frame_rate().get();
 538                         Frame const trim_start = j->trim_start().frames_round (cfr);
 539                         Frame const trim_end = j->trim_end().frames_round (cfr);
 540                         int const ffr = _film->video_frame_rate ();
 541
 542                         DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
 543                         if (j->reference_video ()) {
 544                                 shared_ptr<dcp::ReelAsset> ra = k->main_picture ();
 545                                 DCPOMATIC_ASSERT (ra);
 546                                 ra->set_entry_point (ra->entry_point() + trim_start);
 547                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 548                                 a.push_back (
 549                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 550                                         );
 551                         }
 552
 553                         if (j->reference_audio ()) {
 554                                 shared_ptr<dcp::ReelAsset> ra = k->main_sound ();
 555                                 DCPOMATIC_ASSERT (ra);
 556                                 ra->set_entry_point (ra->entry_point() + trim_start);
 557                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 558                                 a.push_back (
 559                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 560                                         );
 561                         }
 562
 563                         if (j->reference_text (TEXT_OPEN_SUBTITLE)) {
 564                                 shared_ptr<dcp::ReelAsset> ra = k->main_subtitle ();
 565                                 DCPOMATIC_ASSERT (ra);
 566                                 ra->set_entry_point (ra->entry_point() + trim_start);
 567                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 568                                 a.push_back (
 569                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 570                                         );
 571                         }
 572
 573                         if (j->reference_text (TEXT_CLOSED_CAPTION)) {
 574                                 shared_ptr<dcp::ReelAsset> ra = k->closed_caption ();
 575                                 DCPOMATIC_ASSERT (ra);
 576                                 ra->set_entry_point (ra->entry_point() + trim_start);
 577                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 578                                 a.push_back (
 579                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 580                                         );
 581                         }
 582
 583                         /* Assume that main picture duration is the length of the reel */
 584                         offset += k->main_picture()->duration ();
 585                 }
 586         }
 587
 588         return a;
 589 }
 590
 591 bool
 592 Player::pass ()
 593 {
 594         boost::mutex::scoped_lock lm (_mutex);
 595
 596         if (!_have_valid_pieces) {
 597                 /* This should only happen when we are under the control of the butler.  In this case, _have_valid_pieces
 598                    will be false if something in the Player has changed and we are waiting for the butler to notice
 599                    and do a seek back to the place we were at before.  During this time we don't want pass() to do anything,
 600                    as just after setup_pieces the new decoders will be back to time 0 until the seek has gone through.  Just do nothing
 601                    here and assume that the seek will be forthcoming.
 602                 */
 603                 return false;
 604         }
 605
 606         if (_playlist->length() == DCPTime()) {
 607                 /* Special case of an empty Film; just give one black frame */
 608                 emit_video (black_player_video_frame(EYES_BOTH), DCPTime());
 609                 return true;
 610         }
 611
 612         /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
 613
 614         shared_ptr<Piece> earliest_content;
 615         optional<DCPTime> earliest_time;
 616
 617         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 618                 if (i->done) {
 619                         continue;
 620                 }
 621
 622                 DCPTime const t = content_time_to_dcp (i, max(i->decoder->position(), i->content->trim_start()));
 623                 if (t > i->content->end()) {
 624                         i->done = true;
 625                 } else {
 626
 627                         /* Given two choices at the same time, pick the one with texts so we see it before
 628                            the video.
 629                         */
 630                         if (!earliest_time || t < *earliest_time || (t == *earliest_time && !i->decoder->text.empty())) {
 631                                 earliest_time = t;
 632                                 earliest_content = i;
 633                         }
 634                 }
 635         }
 636
 637         bool done = false;
 638
 639         enum {
 640                 NONE,
 641                 CONTENT,
 642                 BLACK,
 643                 SILENT
 644         } which = NONE;
 645
 646         if (earliest_content) {
 647                 which = CONTENT;
 648         }
 649
 650         if (!_black.done() && (!earliest_time || _black.position() < *earliest_time)) {
 651                 earliest_time = _black.position ();
 652                 which = BLACK;
 653         }
 654
 655         if (!_silent.done() && (!earliest_time || _silent.position() < *earliest_time)) {
 656                 earliest_time = _silent.position ();
 657                 which = SILENT;
 658         }
 659
 660         switch (which) {
 661         case CONTENT:
 662                 earliest_content->done = earliest_content->decoder->pass ();
 663                 break;
 664         case BLACK:
 665                 emit_video (black_player_video_frame(EYES_BOTH), _black.position());
 666                 _black.set_position (_black.position() + one_video_frame());
 667                 break;
 668         case SILENT:
 669         {
 670                 DCPTimePeriod period (_silent.period_at_position());
 671                 if (_last_audio_time) {
 672                         /* Sometimes the thing that happened last finishes fractionally before
 673                            this silence.  Bodge the start time of the silence to fix it.  I'm
 674                            not sure if this is the right solution --- maybe the last thing should
 675                            be padded `forward' rather than this thing padding `back'.
 676                         */
 677                         period.from = min(period.from, *_last_audio_time);
 678                 }
 679                 if (period.duration() > one_video_frame()) {
 680                         period.to = period.from + one_video_frame();
 681                 }
 682                 fill_audio (period);
 683                 _silent.set_position (period.to);
 684                 break;
 685         }
 686         case NONE:
 687                 done = true;
 688                 break;
 689         }
 690
 691         /* Emit any audio that is ready */
 692
 693         /* Work out the time before which the audio is definitely all here.  This is the earliest last_push_end of one
 694            of our streams, or the position of the _silent.
 695         */
 696         DCPTime pull_to = _film->length ();
 697         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 698                 if (!i->second.piece->done && i->second.last_push_end < pull_to) {
 699                         pull_to = i->second.last_push_end;
 700                 }
 701         }
 702         if (!_silent.done() && _silent.position() < pull_to) {
 703                 pull_to = _silent.position();
 704         }
 705
 706         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 707         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 708                 if (_last_audio_time && i->second < *_last_audio_time) {
 709                         /* This new data comes before the last we emitted (or the last seek); discard it */
 710                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 711                         if (!cut.first) {
 712                                 continue;
 713                         }
 714                         *i = cut;
 715                 } else if (_last_audio_time && i->second > *_last_audio_time) {
 716                         /* There's a gap between this data and the last we emitted; fill with silence */
 717                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 718                 }
 719
 720                 emit_audio (i->first, i->second);
 721         }
 722
 723         if (done) {
 724                 _shuffler->flush ();
 725                 for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 726                         do_emit_video(i->first, i->second);
 727                 }
 728         }
 729
 730         return done;
 731 }
 732
 733 /** @return Open subtitles for the frame at the given time, converted to images */
 734 optional<PositionImage>
 735 Player::open_subtitles_for_frame (DCPTime time) const
 736 {
 737         list<PositionImage> captions;
 738         int const vfr = _film->video_frame_rate();
 739
 740         BOOST_FOREACH (
 741                 PlayerText j,
 742                 _active_texts[TEXT_OPEN_SUBTITLE].get_burnt(DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), _always_burn_open_subtitles)
 743                 ) {
 744
 745                 /* Bitmap subtitles */
 746                 list<PositionImage> c = transform_bitmap_texts (j.bitmap);
 747                 copy (c.begin(), c.end(), back_inserter (captions));
 748
 749                 /* String subtitles (rendered to an image) */
 750                 if (!j.string.empty ()) {
 751                         list<PositionImage> s = render_text (j.string, j.fonts, _video_container_size, time, vfr);
 752                         copy (s.begin(), s.end(), back_inserter (captions));
 753                 }
 754         }
 755
 756         if (captions.empty ()) {
 757                 return optional<PositionImage> ();
 758         }
 759
 760         return merge (captions);
 761 }
 762
 763 void
 764 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 765 {
 766         shared_ptr<Piece> piece = wp.lock ();
 767         if (!piece) {
 768                 return;
 769         }
 770
 771         FrameRateChange frc(piece->content->active_video_frame_rate(), _film->video_frame_rate());
 772         if (frc.skip && (video.frame % 2) == 1) {
 773                 return;
 774         }
 775
 776         /* Time of the first frame we will emit */
 777         DCPTime const time = content_video_to_dcp (piece, video.frame);
 778
 779         /* Discard if it's before the content's period or the last accurate seek.  We can't discard
 780            if it's after the content's period here as in that case we still need to fill any gap between
 781            `now' and the end of the content's period.
 782         */
 783         if (time < piece->content->position() || (_last_video_time && time < *_last_video_time)) {
 784                 return;
 785         }
 786
 787         /* Fill gaps that we discover now that we have some video which needs to be emitted.
 788            This is where we need to fill to.
 789         */
 790         DCPTime fill_to = min (time, piece->content->end());
 791
 792         if (_last_video_time) {
 793                 DCPTime fill_from = max (*_last_video_time, piece->content->position());
 794                 LastVideoMap::const_iterator last = _last_video.find (wp);
 795                 if (_film->three_d()) {
 796                         Eyes fill_to_eyes = video.eyes;
 797                         if (fill_to == piece->content->end()) {
 798                                 /* Don't fill after the end of the content */
 799                                 fill_to_eyes = EYES_LEFT;
 800                         }
 801                         DCPTime j = fill_from;
 802                         Eyes eyes = _last_video_eyes.get_value_or(EYES_LEFT);
 803                         if (eyes == EYES_BOTH) {
 804                                 eyes = EYES_LEFT;
 805                         }
 806                         while (j < fill_to || eyes != fill_to_eyes) {
 807                                 if (last != _last_video.end()) {
 808                                         shared_ptr<PlayerVideo> copy = last->second->shallow_copy();
 809                                         copy->set_eyes (eyes);
 810                                         emit_video (copy, j);
 811                                 } else {
 812                                         emit_video (black_player_video_frame(eyes), j);
 813                                 }
 814                                 if (eyes == EYES_RIGHT) {
 815                                         j += one_video_frame();
 816                                 }
 817                                 eyes = increment_eyes (eyes);
 818                         }
 819                 } else {
 820                         for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
 821                                 if (last != _last_video.end()) {
 822                                         emit_video (last->second, j);
 823                                 } else {
 824                                         emit_video (black_player_video_frame(EYES_BOTH), j);
 825                                 }
 826                         }
 827                 }
 828         }
 829
 830         _last_video[wp].reset (
 831                 new PlayerVideo (
 832                         video.image,
 833                         piece->content->video->crop (),
 834                         piece->content->video->fade (video.frame),
 835                         piece->content->video->scale().size (
 836                                 piece->content->video, _video_container_size, _film->frame_size ()
 837                                 ),
 838                         _video_container_size,
 839                         video.eyes,
 840                         video.part,
 841                         piece->content->video->colour_conversion(),
 842                         piece->content,
 843                         video.frame
 844                         )
 845                 );
 846
 847         DCPTime t = time;
 848         for (int i = 0; i < frc.repeat; ++i) {
 849                 if (t < piece->content->end()) {
 850                         emit_video (_last_video[wp], t);
 851                 }
 852                 t += one_video_frame ();
 853         }
 854 }
 855
 856 void
 857 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 858 {
 859         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 860
 861         shared_ptr<Piece> piece = wp.lock ();
 862         if (!piece) {
 863                 return;
 864         }
 865
 866         shared_ptr<AudioContent> content = piece->content->audio;
 867         DCPOMATIC_ASSERT (content);
 868
 869         /* Compute time in the DCP */
 870         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame);
 871         /* And the end of this block in the DCP */
 872         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), content->resampled_frame_rate());
 873
 874         /* Remove anything that comes before the start or after the end of the content */
 875         if (time < piece->content->position()) {
 876                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 877                 if (!cut.first) {
 878                         /* This audio is entirely discarded */
 879                         return;
 880                 }
 881                 content_audio.audio = cut.first;
 882                 time = cut.second;
 883         } else if (time > piece->content->end()) {
 884                 /* Discard it all */
 885                 return;
 886         } else if (end > piece->content->end()) {
 887                 Frame const remaining_frames = DCPTime(piece->content->end() - time).frames_round(_film->audio_frame_rate());
 888                 if (remaining_frames == 0) {
 889                         return;
 890                 }
 891                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 892                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 893                 content_audio.audio = cut;
 894         }
 895
 896         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 897
 898         /* Gain */
 899
 900         if (content->gain() != 0) {
 901                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 902                 gain->apply_gain (content->gain ());
 903                 content_audio.audio = gain;
 904         }
 905
 906         /* Remap */
 907
 908         content_audio.audio = remap (content_audio.audio, _film->audio_channels(), stream->mapping());
 909
 910         /* Process */
 911
 912         if (_audio_processor) {
 913                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 914         }
 915
 916         /* Push */
 917
 918         _audio_merger.push (content_audio.audio, time);
 919         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 920         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 921 }
 922
 923 void
 924 Player::bitmap_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentBitmapText subtitle)
 925 {
 926         shared_ptr<Piece> piece = wp.lock ();
 927         shared_ptr<const TextContent> text = wc.lock ();
 928         if (!piece || !text) {
 929                 return;
 930         }
 931
 932         /* Apply content's subtitle offsets */
 933         subtitle.sub.rectangle.x += text->x_offset ();
 934         subtitle.sub.rectangle.y += text->y_offset ();
 935
 936         /* Apply a corrective translation to keep the subtitle centred after the scale that is coming up */
 937         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * ((text->x_scale() - 1) / 2);
 938         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * ((text->y_scale() - 1) / 2);
 939
 940         /* Apply content's subtitle scale */
 941         subtitle.sub.rectangle.width *= text->x_scale ();
 942         subtitle.sub.rectangle.height *= text->y_scale ();
 943
 944         PlayerText ps;
 945         ps.bitmap.push_back (subtitle.sub);
 946         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 947
 948         _active_texts[subtitle.type()].add_from (wc, ps, from);
 949 }
 950
 951 void
 952 Player::plain_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentStringText subtitle)
 953 {
 954         shared_ptr<Piece> piece = wp.lock ();
 955         shared_ptr<const TextContent> text = wc.lock ();
 956         if (!piece || !text) {
 957                 return;
 958         }
 959
 960         PlayerText ps;
 961         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 962
 963         if (from > piece->content->end()) {
 964                 return;
 965         }
 966
 967         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 968                 s.set_h_position (s.h_position() + text->x_offset ());
 969                 s.set_v_position (s.v_position() + text->y_offset ());
 970                 float const xs = text->x_scale();
 971                 float const ys = text->y_scale();
 972                 float size = s.size();
 973
 974                 /* Adjust size to express the common part of the scaling;
 975                    e.g. if xs = ys = 0.5 we scale size by 2.
 976                 */
 977                 if (xs > 1e-5 && ys > 1e-5) {
 978                         size *= 1 / min (1 / xs, 1 / ys);
 979                 }
 980                 s.set_size (size);
 981
 982                 /* Then express aspect ratio changes */
 983                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 984                         s.set_aspect_adjust (xs / ys);
 985                 }
 986
 987                 s.set_in (dcp::Time(from.seconds(), 1000));
 988                 ps.string.push_back (StringText (s, text->outline_width()));
 989                 ps.add_fonts (text->fonts ());
 990         }
 991
 992         _active_texts[subtitle.type()].add_from (wc, ps, from);
 993 }
 994
 995 void
 996 Player::subtitle_stop (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentTime to, TextType type)
 997 {
 998         if (!_active_texts[type].have (wc)) {
 999                 return;
1000         }
1001
1002         shared_ptr<Piece> piece = wp.lock ();
1003         shared_ptr<const TextContent> text = wc.lock ();
1004         if (!piece || !text) {
1005                 return;
1006         }
1007
1008         DCPTime const dcp_to = content_time_to_dcp (piece, to);
1009
1010         if (dcp_to > piece->content->end()) {
1011                 return;
1012         }
1013
1014         pair<PlayerText, DCPTime> from = _active_texts[type].add_to (wc, dcp_to);
1015
1016         bool const always = (type == TEXT_OPEN_SUBTITLE && _always_burn_open_subtitles);
1017         if (text->use() && !always && !text->burn()) {
1018                 Text (from.first, type, DCPTimePeriod (from.second, dcp_to));
1019         }
1020 }
1021
1022 void
1023 Player::seek (DCPTime time, bool accurate)
1024 {
1025         boost::mutex::scoped_lock lm (_mutex);
1026
1027         if (!_have_valid_pieces) {
1028                 setup_pieces ();
1029         }
1030
1031         if (_shuffler) {
1032                 _shuffler->clear ();
1033         }
1034
1035         _delay.clear ();
1036
1037         if (_audio_processor) {
1038                 _audio_processor->flush ();
1039         }
1040
1041         _audio_merger.clear ();
1042         for (int i = 0; i < TEXT_COUNT; ++i) {
1043                 _active_texts[i].clear ();
1044         }
1045
1046         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1047                 if (time < i->content->position()) {
1048                         /* Before; seek to the start of the content */
1049                         i->decoder->seek (dcp_to_content_time (i, i->content->position()), accurate);
1050                         i->done = false;
1051                 } else if (i->content->position() <= time && time < i->content->end()) {
1052                         /* During; seek to position */
1053                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
1054                         i->done = false;
1055                 } else {
1056                         /* After; this piece is done */
1057                         i->done = true;
1058                 }
1059         }
1060
1061         if (accurate) {
1062                 _last_video_time = time;
1063                 _last_video_eyes = EYES_LEFT;
1064                 _last_audio_time = time;
1065         } else {
1066                 _last_video_time = optional<DCPTime>();
1067                 _last_video_eyes = optional<Eyes>();
1068                 _last_audio_time = optional<DCPTime>();
1069         }
1070
1071         _black.set_position (time);
1072         _silent.set_position (time);
1073
1074         _last_video.clear ();
1075 }
1076
1077 void
1078 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1079 {
1080         /* We need a delay to give a little wiggle room to ensure that relevent subtitles arrive at the
1081            player before the video that requires them.
1082         */
1083         _delay.push_back (make_pair (pv, time));
1084
1085         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1086                 _last_video_time = time + one_video_frame();
1087         }
1088         _last_video_eyes = increment_eyes (pv->eyes());
1089
1090         if (_delay.size() < 3) {
1091                 return;
1092         }
1093
1094         pair<shared_ptr<PlayerVideo>, DCPTime> to_do = _delay.front();
1095         _delay.pop_front();
1096         do_emit_video (to_do.first, to_do.second);
1097 }
1098
1099 void
1100 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1101 {
1102         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1103                 for (int i = 0; i < TEXT_COUNT; ++i) {
1104                         _active_texts[i].clear_before (time);
1105                 }
1106         }
1107
1108         optional<PositionImage> subtitles = open_subtitles_for_frame (time);
1109         if (subtitles) {
1110                 pv->set_text (subtitles.get ());
1111         }
1112
1113         Video (pv, time);
1114 }
1115
1116 void
1117 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1118 {
1119         /* Log if the assert below is about to fail */
1120         if (_last_audio_time && time != *_last_audio_time) {
1121                 _film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_last_audio_time)), LogEntry::TYPE_WARNING);
1122         }
1123
1124         /* This audio must follow on from the previous */
1125         DCPOMATIC_ASSERT (!_last_audio_time || time == *_last_audio_time);
1126         Audio (data, time);
1127         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate());
1128 }
1129
1130 void
1131 Player::fill_audio (DCPTimePeriod period)
1132 {
1133         if (period.from == period.to) {
1134                 return;
1135         }
1136
1137         DCPOMATIC_ASSERT (period.from < period.to);
1138
1139         DCPTime t = period.from;
1140         while (t < period.to) {
1141                 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1142                 Frame const samples = block.frames_round(_film->audio_frame_rate());
1143                 if (samples) {
1144                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1145                         silence->make_silent ();
1146                         emit_audio (silence, t);
1147                 }
1148                 t += block;
1149         }
1150 }
1151
1152 DCPTime
1153 Player::one_video_frame () const
1154 {
1155         return DCPTime::from_frames (1, _film->video_frame_rate ());
1156 }
1157
1158 pair<shared_ptr<AudioBuffers>, DCPTime>
1159 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1160 {
1161         DCPTime const discard_time = discard_to - time;
1162         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1163         Frame remaining_frames = audio->frames() - discard_frames;
1164         if (remaining_frames <= 0) {
1165                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1166         }
1167         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1168         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1169         return make_pair(cut, time + discard_time);
1170 }
1171
1172 void
1173 Player::set_dcp_decode_reduction (optional<int> reduction)
1174 {
1175         {
1176                 boost::mutex::scoped_lock lm (_mutex);
1177
1178                 if (reduction == _dcp_decode_reduction) {
1179                         return;
1180                 }
1181
1182                 _dcp_decode_reduction = reduction;
1183                 _have_valid_pieces = false;
1184         }
1185
1186         Changed (PlayerProperty::DCP_DECODE_REDUCTION, false);
1187 }
1188
1189 DCPTime
1190 Player::content_time_to_dcp (shared_ptr<Content> content, ContentTime t)
1191 {
1192         boost::mutex::scoped_lock lm (_mutex);
1193
1194         if (_have_valid_pieces) {
1195                 setup_pieces ();
1196         }
1197
1198         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1199                 if (i->content == content) {
1200                         return content_time_to_dcp (i, t);
1201                 }
1202         }
1203
1204         DCPOMATIC_ASSERT (false);
1205         return DCPTime ();
1206 }