src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2018 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_text.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "text_content.h"
  44 #include "text_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "dcp_decoder.h"
  48 #include "image_decoder.h"
  49 #include "compose.hpp"
  50 #include "shuffler.h"
  51 #include <dcp/reel.h>
  52 #include <dcp/reel_sound_asset.h>
  53 #include <dcp/reel_subtitle_asset.h>
  54 #include <dcp/reel_picture_asset.h>
  55 #include <dcp/reel_closed_caption_asset.h>
  56 #include <boost/foreach.hpp>
  57 #include <stdint.h>
  58 #include <algorithm>
  59 #include <iostream>
  60
  61 #include "i18n.h"
  62
  63 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
  64
  65 using std::list;
  66 using std::cout;
  67 using std::min;
  68 using std::max;
  69 using std::min;
  70 using std::vector;
  71 using std::pair;
  72 using std::map;
  73 using std::make_pair;
  74 using std::copy;
  75 using boost::shared_ptr;
  76 using boost::weak_ptr;
  77 using boost::dynamic_pointer_cast;
  78 using boost::optional;
  79 using boost::scoped_ptr;
  80
  81 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
  82 int const PlayerProperty::PLAYLIST = 701;
  83 int const PlayerProperty::FILM_CONTAINER = 702;
  84 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
  85 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
  86
  87 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  88         : _film (film)
  89         , _playlist (playlist)
  90         , _have_valid_pieces (false)
  91         , _ignore_video (false)
  92         , _ignore_caption (false)
  93         , _fast (false)
  94         , _play_referenced (false)
  95         , _audio_merger (_film->audio_frame_rate())
  96         , _shuffler (0)
  97 {
  98         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  99         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 100         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
 101         set_video_container_size (_film->frame_size ());
 102
 103         film_changed (Film::AUDIO_PROCESSOR);
 104
 105         seek (DCPTime (), true);
 106 }
 107
 108 Player::~Player ()
 109 {
 110         delete _shuffler;
 111 }
 112
 113 void
 114 Player::setup_pieces ()
 115 {
 116         _pieces.clear ();
 117
 118         delete _shuffler;
 119         _shuffler = new Shuffler();
 120         _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
 121
 122         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 123
 124                 if (!i->paths_valid ()) {
 125                         continue;
 126                 }
 127
 128                 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log(), _fast);
 129                 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
 130
 131                 if (!decoder) {
 132                         /* Not something that we can decode; e.g. Atmos content */
 133                         continue;
 134                 }
 135
 136                 if (decoder->video && _ignore_video) {
 137                         decoder->video->set_ignore (true);
 138                 }
 139
 140                 if (_ignore_caption) {
 141                         BOOST_FOREACH (shared_ptr<TextDecoder> i, decoder->caption) {
 142                                 i->set_ignore (true);
 143                         }
 144                 }
 145
 146                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 147                 if (dcp) {
 148                         dcp->set_decode_referenced (_play_referenced);
 149                         if (_play_referenced) {
 150                                 dcp->set_forced_reduction (_dcp_decode_reduction);
 151                         }
 152                 }
 153
 154                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 155                 _pieces.push_back (piece);
 156
 157                 if (decoder->video) {
 158                         if (i->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT || i->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) {
 159                                 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
 160                                 decoder->video->Data.connect (bind (&Shuffler::video, _shuffler, weak_ptr<Piece>(piece), _1));
 161                         } else {
 162                                 decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece>(piece), _1));
 163                         }
 164                 }
 165
 166                 if (decoder->audio) {
 167                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 168                 }
 169
 170                 list<shared_ptr<TextDecoder> >::const_iterator j = decoder->caption.begin();
 171
 172                 while (j != decoder->caption.end()) {
 173                         (*j)->BitmapStart.connect (
 174                                 bind(&Player::bitmap_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 175                                 );
 176                         (*j)->PlainStart.connect (
 177                                 bind(&Player::plain_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 178                                 );
 179                         (*j)->Stop.connect (
 180                                 bind(&Player::subtitle_stop, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1, _2)
 181                                 );
 182
 183                         ++j;
 184                 }
 185         }
 186
 187         _stream_states.clear ();
 188         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 189                 if (i->content->audio) {
 190                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 191                                 _stream_states[j] = StreamState (i, i->content->position ());
 192                         }
 193                 }
 194         }
 195
 196         _black = Empty (_film->content(), _film->length(), bind(&Content::video, _1));
 197         _silent = Empty (_film->content(), _film->length(), bind(&Content::audio, _1));
 198
 199         _last_video_time = DCPTime ();
 200         _last_video_eyes = EYES_BOTH;
 201         _last_audio_time = DCPTime ();
 202         _have_valid_pieces = true;
 203 }
 204
 205 void
 206 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
 207 {
 208         shared_ptr<Content> c = w.lock ();
 209         if (!c) {
 210                 return;
 211         }
 212
 213         if (
 214                 property == ContentProperty::POSITION ||
 215                 property == ContentProperty::LENGTH ||
 216                 property == ContentProperty::TRIM_START ||
 217                 property == ContentProperty::TRIM_END ||
 218                 property == ContentProperty::PATH ||
 219                 property == VideoContentProperty::FRAME_TYPE ||
 220                 property == VideoContentProperty::COLOUR_CONVERSION ||
 221                 property == AudioContentProperty::STREAMS ||
 222                 property == DCPContentProperty::NEEDS_ASSETS ||
 223                 property == DCPContentProperty::NEEDS_KDM ||
 224                 property == TextContentProperty::COLOUR ||
 225                 property == TextContentProperty::EFFECT ||
 226                 property == TextContentProperty::EFFECT_COLOUR ||
 227                 property == FFmpegContentProperty::SUBTITLE_STREAM ||
 228                 property == FFmpegContentProperty::FILTERS
 229                 ) {
 230
 231                 _have_valid_pieces = false;
 232                 Changed (property, frequent);
 233
 234         } else if (
 235                 property == TextContentProperty::LINE_SPACING ||
 236                 property == TextContentProperty::OUTLINE_WIDTH ||
 237                 property == TextContentProperty::Y_SCALE ||
 238                 property == TextContentProperty::FADE_IN ||
 239                 property == TextContentProperty::FADE_OUT ||
 240                 property == ContentProperty::VIDEO_FRAME_RATE ||
 241                 property == TextContentProperty::USE ||
 242                 property == TextContentProperty::X_OFFSET ||
 243                 property == TextContentProperty::Y_OFFSET ||
 244                 property == TextContentProperty::X_SCALE ||
 245                 property == TextContentProperty::FONTS ||
 246                 property == TextContentProperty::TYPE ||
 247                 property == VideoContentProperty::CROP ||
 248                 property == VideoContentProperty::SCALE ||
 249                 property == VideoContentProperty::FADE_IN ||
 250                 property == VideoContentProperty::FADE_OUT
 251                 ) {
 252
 253                 Changed (property, frequent);
 254         }
 255 }
 256
 257 void
 258 Player::set_video_container_size (dcp::Size s)
 259 {
 260         if (s == _video_container_size) {
 261                 return;
 262         }
 263
 264         _video_container_size = s;
 265
 266         _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 267         _black_image->make_black ();
 268
 269         Changed (PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 270 }
 271
 272 void
 273 Player::playlist_changed ()
 274 {
 275         _have_valid_pieces = false;
 276         Changed (PlayerProperty::PLAYLIST, false);
 277 }
 278
 279 void
 280 Player::film_changed (Film::Property p)
 281 {
 282         /* Here we should notice Film properties that affect our output, and
 283            alert listeners that our output now would be different to how it was
 284            last time we were run.
 285         */
 286
 287         if (p == Film::CONTAINER) {
 288                 Changed (PlayerProperty::FILM_CONTAINER, false);
 289         } else if (p == Film::VIDEO_FRAME_RATE) {
 290                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 291                    so we need new pieces here.
 292                 */
 293                 _have_valid_pieces = false;
 294                 Changed (PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
 295         } else if (p == Film::AUDIO_PROCESSOR) {
 296                 if (_film->audio_processor ()) {
 297                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 298                 }
 299         } else if (p == Film::AUDIO_CHANNELS) {
 300                 _audio_merger.clear ();
 301         }
 302 }
 303
 304 list<PositionImage>
 305 Player::transform_bitmap_captions (list<BitmapText> subs) const
 306 {
 307         list<PositionImage> all;
 308
 309         for (list<BitmapText>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 310                 if (!i->image) {
 311                         continue;
 312                 }
 313
 314                 /* We will scale the subtitle up to fit _video_container_size */
 315                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
 316
 317                 all.push_back (
 318                         PositionImage (
 319                                 i->image->scale (
 320                                         scaled_size,
 321                                         dcp::YUV_TO_RGB_REC601,
 322                                         i->image->pixel_format (),
 323                                         true,
 324                                         _fast
 325                                         ),
 326                                 Position<int> (
 327                                         lrint (_video_container_size.width * i->rectangle.x),
 328                                         lrint (_video_container_size.height * i->rectangle.y)
 329                                         )
 330                                 )
 331                         );
 332         }
 333
 334         return all;
 335 }
 336
 337 shared_ptr<PlayerVideo>
 338 Player::black_player_video_frame (Eyes eyes) const
 339 {
 340         return shared_ptr<PlayerVideo> (
 341                 new PlayerVideo (
 342                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 343                         Crop (),
 344                         optional<double> (),
 345                         _video_container_size,
 346                         _video_container_size,
 347                         eyes,
 348                         PART_WHOLE,
 349                         PresetColourConversion::all().front().conversion,
 350                         boost::weak_ptr<Content>(),
 351                         boost::optional<Frame>()
 352                 )
 353         );
 354 }
 355
 356 Frame
 357 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 358 {
 359         DCPTime s = t - piece->content->position ();
 360         s = min (piece->content->length_after_trim(), s);
 361         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 362
 363         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 364            then convert that ContentTime to frames at the content's rate.  However this fails for
 365            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 366            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 367
 368            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 369         */
 370         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 371 }
 372
 373 DCPTime
 374 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 375 {
 376         /* See comment in dcp_to_content_video */
 377         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime(piece->content->trim_start(), piece->frc);
 378         return d + piece->content->position();
 379 }
 380
 381 Frame
 382 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 383 {
 384         DCPTime s = t - piece->content->position ();
 385         s = min (piece->content->length_after_trim(), s);
 386         /* See notes in dcp_to_content_video */
 387         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 388 }
 389
 390 DCPTime
 391 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 392 {
 393         /* See comment in dcp_to_content_video */
 394         return DCPTime::from_frames (f, _film->audio_frame_rate())
 395                 - DCPTime (piece->content->trim_start(), piece->frc)
 396                 + piece->content->position();
 397 }
 398
 399 ContentTime
 400 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 401 {
 402         DCPTime s = t - piece->content->position ();
 403         s = min (piece->content->length_after_trim(), s);
 404         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 405 }
 406
 407 DCPTime
 408 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 409 {
 410         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 411 }
 412
 413 list<shared_ptr<Font> >
 414 Player::get_subtitle_fonts ()
 415 {
 416         if (!_have_valid_pieces) {
 417                 setup_pieces ();
 418         }
 419
 420         list<shared_ptr<Font> > fonts;
 421         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 422                 BOOST_FOREACH (shared_ptr<TextContent> j, i->content->caption) {
 423                         /* XXX: things may go wrong if there are duplicate font IDs
 424                            with different font files.
 425                         */
 426                         list<shared_ptr<Font> > f = j->fonts ();
 427                         copy (f.begin(), f.end(), back_inserter (fonts));
 428                 }
 429         }
 430
 431         return fonts;
 432 }
 433
 434 /** Set this player never to produce any video data */
 435 void
 436 Player::set_ignore_video ()
 437 {
 438         _ignore_video = true;
 439 }
 440
 441 void
 442 Player::set_ignore_caption ()
 443 {
 444         _ignore_caption = true;
 445 }
 446
 447 /** Set the player to always burn open captions into the image regardless of the content settings */
 448 void
 449 Player::set_always_burn_open_captions ()
 450 {
 451         _always_burn_open_captions = true;
 452 }
 453
 454 /** Sets up the player to be faster, possibly at the expense of quality */
 455 void
 456 Player::set_fast ()
 457 {
 458         _fast = true;
 459         _have_valid_pieces = false;
 460 }
 461
 462 void
 463 Player::set_play_referenced ()
 464 {
 465         _play_referenced = true;
 466         _have_valid_pieces = false;
 467 }
 468
 469 list<ReferencedReelAsset>
 470 Player::get_reel_assets ()
 471 {
 472         list<ReferencedReelAsset> a;
 473
 474         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 475                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 476                 if (!j) {
 477                         continue;
 478                 }
 479
 480                 scoped_ptr<DCPDecoder> decoder;
 481                 try {
 482                         decoder.reset (new DCPDecoder (j, _film->log(), false));
 483                 } catch (...) {
 484                         return a;
 485                 }
 486
 487                 int64_t offset = 0;
 488                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 489
 490                         DCPOMATIC_ASSERT (j->video_frame_rate ());
 491                         double const cfr = j->video_frame_rate().get();
 492                         Frame const trim_start = j->trim_start().frames_round (cfr);
 493                         Frame const trim_end = j->trim_end().frames_round (cfr);
 494                         int const ffr = _film->video_frame_rate ();
 495
 496                         DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
 497                         if (j->reference_video ()) {
 498                                 shared_ptr<dcp::ReelAsset> ra = k->main_picture ();
 499                                 DCPOMATIC_ASSERT (ra);
 500                                 ra->set_entry_point (ra->entry_point() + trim_start);
 501                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 502                                 a.push_back (
 503                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 504                                         );
 505                         }
 506
 507                         if (j->reference_audio ()) {
 508                                 shared_ptr<dcp::ReelAsset> ra = k->main_sound ();
 509                                 DCPOMATIC_ASSERT (ra);
 510                                 ra->set_entry_point (ra->entry_point() + trim_start);
 511                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 512                                 a.push_back (
 513                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 514                                         );
 515                         }
 516
 517                         if (j->reference_caption (CAPTION_OPEN)) {
 518                                 shared_ptr<dcp::ReelAsset> ra = k->main_subtitle ();
 519                                 DCPOMATIC_ASSERT (ra);
 520                                 ra->set_entry_point (ra->entry_point() + trim_start);
 521                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 522                                 a.push_back (
 523                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 524                                         );
 525                         }
 526
 527                         if (j->reference_caption (CAPTION_CLOSED)) {
 528                                 shared_ptr<dcp::ReelAsset> ra = k->closed_caption ();
 529                                 DCPOMATIC_ASSERT (ra);
 530                                 ra->set_entry_point (ra->entry_point() + trim_start);
 531                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 532                                 a.push_back (
 533                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 534                                         );
 535                         }
 536
 537                         /* Assume that main picture duration is the length of the reel */
 538                         offset += k->main_picture()->duration ();
 539                 }
 540         }
 541
 542         return a;
 543 }
 544
 545 bool
 546 Player::pass ()
 547 {
 548         if (!_have_valid_pieces) {
 549                 setup_pieces ();
 550         }
 551
 552         if (_playlist->length() == DCPTime()) {
 553                 /* Special case of an empty Film; just give one black frame */
 554                 emit_video (black_player_video_frame(EYES_BOTH), DCPTime());
 555                 return true;
 556         }
 557
 558         /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
 559
 560         shared_ptr<Piece> earliest_content;
 561         optional<DCPTime> earliest_time;
 562
 563         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 564                 if (i->done) {
 565                         continue;
 566                 }
 567
 568                 DCPTime const t = content_time_to_dcp (i, max(i->decoder->position(), i->content->trim_start()));
 569                 if (t > i->content->end()) {
 570                         i->done = true;
 571                 } else {
 572
 573                         /* Given two choices at the same time, pick the one with captions so we see it before
 574                            the video.
 575                         */
 576                         if (!earliest_time || t < *earliest_time || (t == *earliest_time && !i->decoder->caption.empty())) {
 577                                 earliest_time = t;
 578                                 earliest_content = i;
 579                         }
 580                 }
 581         }
 582
 583         bool done = false;
 584
 585         enum {
 586                 NONE,
 587                 CONTENT,
 588                 BLACK,
 589                 SILENT
 590         } which = NONE;
 591
 592         if (earliest_content) {
 593                 which = CONTENT;
 594         }
 595
 596         if (!_black.done() && (!earliest_time || _black.position() < *earliest_time)) {
 597                 earliest_time = _black.position ();
 598                 which = BLACK;
 599         }
 600
 601         if (!_silent.done() && (!earliest_time || _silent.position() < *earliest_time)) {
 602                 earliest_time = _silent.position ();
 603                 which = SILENT;
 604         }
 605
 606         switch (which) {
 607         case CONTENT:
 608                 earliest_content->done = earliest_content->decoder->pass ();
 609                 break;
 610         case BLACK:
 611                 emit_video (black_player_video_frame(EYES_BOTH), _black.position());
 612                 _black.set_position (_black.position() + one_video_frame());
 613                 break;
 614         case SILENT:
 615         {
 616                 DCPTimePeriod period (_silent.period_at_position());
 617                 if (_last_audio_time) {
 618                         /* Sometimes the thing that happened last finishes fractionally before
 619                            this silence.  Bodge the start time of the silence to fix it.  I'm
 620                            not sure if this is the right solution --- maybe the last thing should
 621                            be padded `forward' rather than this thing padding `back'.
 622                         */
 623                         period.from = min(period.from, *_last_audio_time);
 624                 }
 625                 if (period.duration() > one_video_frame()) {
 626                         period.to = period.from + one_video_frame();
 627                 }
 628                 fill_audio (period);
 629                 _silent.set_position (period.to);
 630                 break;
 631         }
 632         case NONE:
 633                 done = true;
 634                 break;
 635         }
 636
 637         /* Emit any audio that is ready */
 638
 639         /* Work out the time before which the audio is definitely all here.  This is the earliest last_push_end of one
 640            of our streams, or the position of the _silent.
 641         */
 642         DCPTime pull_to = _film->length ();
 643         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 644                 if (!i->second.piece->done && i->second.last_push_end < pull_to) {
 645                         pull_to = i->second.last_push_end;
 646                 }
 647         }
 648         if (!_silent.done() && _silent.position() < pull_to) {
 649                 pull_to = _silent.position();
 650         }
 651
 652         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 653         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 654                 if (_last_audio_time && i->second < *_last_audio_time) {
 655                         /* This new data comes before the last we emitted (or the last seek); discard it */
 656                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 657                         if (!cut.first) {
 658                                 continue;
 659                         }
 660                         *i = cut;
 661                 } else if (_last_audio_time && i->second > *_last_audio_time) {
 662                         /* There's a gap between this data and the last we emitted; fill with silence */
 663                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 664                 }
 665
 666                 emit_audio (i->first, i->second);
 667         }
 668
 669         if (done) {
 670                 _shuffler->flush ();
 671                 for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 672                         do_emit_video(i->first, i->second);
 673                 }
 674         }
 675
 676         return done;
 677 }
 678
 679 list<PlayerText>
 680 Player::closed_captions_for_frame (DCPTime time) const
 681 {
 682         return _active_captions[CAPTION_CLOSED].get (
 683                 DCPTimePeriod(time, time + DCPTime::from_frames(1, _film->video_frame_rate()))
 684                 );
 685 }
 686
 687 /** @return Open captions for the frame at the given time, converted to images */
 688 optional<PositionImage>
 689 Player::open_captions_for_frame (DCPTime time) const
 690 {
 691         list<PositionImage> captions;
 692         int const vfr = _film->video_frame_rate();
 693
 694         BOOST_FOREACH (
 695                 PlayerText j,
 696                 _active_captions[CAPTION_OPEN].get_burnt(DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), _always_burn_open_captions)
 697                 ) {
 698
 699                 /* Image subtitles */
 700                 list<PositionImage> c = transform_bitmap_captions (j.image);
 701                 copy (c.begin(), c.end(), back_inserter (captions));
 702
 703                 /* Text subtitles (rendered to an image) */
 704                 if (!j.text.empty ()) {
 705                         list<PositionImage> s = render_text (j.text, j.fonts, _video_container_size, time, vfr);
 706                         copy (s.begin(), s.end(), back_inserter (captions));
 707                 }
 708         }
 709
 710         if (captions.empty ()) {
 711                 return optional<PositionImage> ();
 712         }
 713
 714         return merge (captions);
 715 }
 716
 717 void
 718 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 719 {
 720         shared_ptr<Piece> piece = wp.lock ();
 721         if (!piece) {
 722                 return;
 723         }
 724
 725         FrameRateChange frc(piece->content->active_video_frame_rate(), _film->video_frame_rate());
 726         if (frc.skip && (video.frame % 2) == 1) {
 727                 return;
 728         }
 729
 730         /* Time of the first frame we will emit */
 731         DCPTime const time = content_video_to_dcp (piece, video.frame);
 732
 733         /* Discard if it's before the content's period or the last accurate seek.  We can't discard
 734            if it's after the content's period here as in that case we still need to fill any gap between
 735            `now' and the end of the content's period.
 736         */
 737         if (time < piece->content->position() || (_last_video_time && time < *_last_video_time)) {
 738                 return;
 739         }
 740
 741         /* Fill gaps that we discover now that we have some video which needs to be emitted.
 742            This is where we need to fill to.
 743         */
 744         DCPTime fill_to = min (time, piece->content->end());
 745
 746         if (_last_video_time) {
 747                 DCPTime fill_from = max (*_last_video_time, piece->content->position());
 748                 LastVideoMap::const_iterator last = _last_video.find (wp);
 749                 if (_film->three_d()) {
 750                         DCPTime j = fill_from;
 751                         Eyes eyes = _last_video_eyes.get_value_or(EYES_LEFT);
 752                         if (eyes == EYES_BOTH) {
 753                                 eyes = EYES_LEFT;
 754                         }
 755                         while (j < fill_to || eyes != video.eyes) {
 756                                 if (last != _last_video.end()) {
 757                                         shared_ptr<PlayerVideo> copy = last->second->shallow_copy();
 758                                         copy->set_eyes (eyes);
 759                                         emit_video (copy, j);
 760                                 } else {
 761                                         emit_video (black_player_video_frame(eyes), j);
 762                                 }
 763                                 if (eyes == EYES_RIGHT) {
 764                                         j += one_video_frame();
 765                                 }
 766                                 eyes = increment_eyes (eyes);
 767                         }
 768                 } else {
 769                         for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
 770                                 if (last != _last_video.end()) {
 771                                         emit_video (last->second, j);
 772                                 } else {
 773                                         emit_video (black_player_video_frame(EYES_BOTH), j);
 774                                 }
 775                         }
 776                 }
 777         }
 778
 779         _last_video[wp].reset (
 780                 new PlayerVideo (
 781                         video.image,
 782                         piece->content->video->crop (),
 783                         piece->content->video->fade (video.frame),
 784                         piece->content->video->scale().size (
 785                                 piece->content->video, _video_container_size, _film->frame_size ()
 786                                 ),
 787                         _video_container_size,
 788                         video.eyes,
 789                         video.part,
 790                         piece->content->video->colour_conversion(),
 791                         piece->content,
 792                         video.frame
 793                         )
 794                 );
 795
 796         DCPTime t = time;
 797         for (int i = 0; i < frc.repeat; ++i) {
 798                 if (t < piece->content->end()) {
 799                         emit_video (_last_video[wp], t);
 800                 }
 801                 t += one_video_frame ();
 802         }
 803 }
 804
 805 void
 806 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 807 {
 808         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 809
 810         shared_ptr<Piece> piece = wp.lock ();
 811         if (!piece) {
 812                 return;
 813         }
 814
 815         shared_ptr<AudioContent> content = piece->content->audio;
 816         DCPOMATIC_ASSERT (content);
 817
 818         /* Compute time in the DCP */
 819         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame);
 820         /* And the end of this block in the DCP */
 821         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), content->resampled_frame_rate());
 822
 823         /* Remove anything that comes before the start or after the end of the content */
 824         if (time < piece->content->position()) {
 825                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 826                 if (!cut.first) {
 827                         /* This audio is entirely discarded */
 828                         return;
 829                 }
 830                 content_audio.audio = cut.first;
 831                 time = cut.second;
 832         } else if (time > piece->content->end()) {
 833                 /* Discard it all */
 834                 return;
 835         } else if (end > piece->content->end()) {
 836                 Frame const remaining_frames = DCPTime(piece->content->end() - time).frames_round(_film->audio_frame_rate());
 837                 if (remaining_frames == 0) {
 838                         return;
 839                 }
 840                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 841                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 842                 content_audio.audio = cut;
 843         }
 844
 845         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 846
 847         /* Gain */
 848
 849         if (content->gain() != 0) {
 850                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 851                 gain->apply_gain (content->gain ());
 852                 content_audio.audio = gain;
 853         }
 854
 855         /* Remap */
 856
 857         content_audio.audio = remap (content_audio.audio, _film->audio_channels(), stream->mapping());
 858
 859         /* Process */
 860
 861         if (_audio_processor) {
 862                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 863         }
 864
 865         /* Push */
 866
 867         _audio_merger.push (content_audio.audio, time);
 868         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 869         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 870 }
 871
 872 void
 873 Player::bitmap_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentBitmapText subtitle)
 874 {
 875         shared_ptr<Piece> piece = wp.lock ();
 876         shared_ptr<const TextContent> caption = wc.lock ();
 877         if (!piece || !caption) {
 878                 return;
 879         }
 880
 881         /* Apply content's subtitle offsets */
 882         subtitle.sub.rectangle.x += caption->x_offset ();
 883         subtitle.sub.rectangle.y += caption->y_offset ();
 884
 885         /* Apply a corrective translation to keep the subtitle centred after the scale that is coming up */
 886         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * ((caption->x_scale() - 1) / 2);
 887         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * ((caption->y_scale() - 1) / 2);
 888
 889         /* Apply content's subtitle scale */
 890         subtitle.sub.rectangle.width *= caption->x_scale ();
 891         subtitle.sub.rectangle.height *= caption->y_scale ();
 892
 893         PlayerText ps;
 894         ps.image.push_back (subtitle.sub);
 895         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 896
 897         _active_captions[subtitle.type()].add_from (wc, ps, from);
 898 }
 899
 900 void
 901 Player::plain_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentStringText subtitle)
 902 {
 903         shared_ptr<Piece> piece = wp.lock ();
 904         shared_ptr<const TextContent> caption = wc.lock ();
 905         if (!piece || !caption) {
 906                 return;
 907         }
 908
 909         PlayerText ps;
 910         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 911
 912         if (from > piece->content->end()) {
 913                 return;
 914         }
 915
 916         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 917                 s.set_h_position (s.h_position() + caption->x_offset ());
 918                 s.set_v_position (s.v_position() + caption->y_offset ());
 919                 float const xs = caption->x_scale();
 920                 float const ys = caption->y_scale();
 921                 float size = s.size();
 922
 923                 /* Adjust size to express the common part of the scaling;
 924                    e.g. if xs = ys = 0.5 we scale size by 2.
 925                 */
 926                 if (xs > 1e-5 && ys > 1e-5) {
 927                         size *= 1 / min (1 / xs, 1 / ys);
 928                 }
 929                 s.set_size (size);
 930
 931                 /* Then express aspect ratio changes */
 932                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 933                         s.set_aspect_adjust (xs / ys);
 934                 }
 935
 936                 s.set_in (dcp::Time(from.seconds(), 1000));
 937                 ps.text.push_back (StringText (s, caption->outline_width()));
 938                 ps.add_fonts (caption->fonts ());
 939         }
 940
 941         _active_captions[subtitle.type()].add_from (wc, ps, from);
 942 }
 943
 944 void
 945 Player::subtitle_stop (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentTime to, TextType type)
 946 {
 947         if (!_active_captions[type].have (wc)) {
 948                 return;
 949         }
 950
 951         shared_ptr<Piece> piece = wp.lock ();
 952         shared_ptr<const TextContent> caption = wc.lock ();
 953         if (!piece || !caption) {
 954                 return;
 955         }
 956
 957         DCPTime const dcp_to = content_time_to_dcp (piece, to);
 958
 959         if (dcp_to > piece->content->end()) {
 960                 return;
 961         }
 962
 963         pair<PlayerText, DCPTime> from = _active_captions[type].add_to (wc, dcp_to);
 964
 965         bool const always = type == CAPTION_OPEN && _always_burn_open_captions;
 966         if (caption->use() && !always && !caption->burn()) {
 967                 Caption (from.first, type, DCPTimePeriod (from.second, dcp_to));
 968         }
 969 }
 970
 971 void
 972 Player::seek (DCPTime time, bool accurate)
 973 {
 974         if (!_have_valid_pieces) {
 975                 setup_pieces ();
 976         }
 977
 978         if (_shuffler) {
 979                 _shuffler->clear ();
 980         }
 981
 982         _delay.clear ();
 983
 984         if (_audio_processor) {
 985                 _audio_processor->flush ();
 986         }
 987
 988         _audio_merger.clear ();
 989         for (int i = 0; i < CAPTION_COUNT; ++i) {
 990                 _active_captions[i].clear ();
 991         }
 992
 993         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 994                 if (time < i->content->position()) {
 995                         /* Before; seek to the start of the content */
 996                         i->decoder->seek (dcp_to_content_time (i, i->content->position()), accurate);
 997                         i->done = false;
 998                 } else if (i->content->position() <= time && time < i->content->end()) {
 999                         /* During; seek to position */
1000                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
1001                         i->done = false;
1002                 } else {
1003                         /* After; this piece is done */
1004                         i->done = true;
1005                 }
1006         }
1007
1008         if (accurate) {
1009                 _last_video_time = time;
1010                 _last_video_eyes = EYES_LEFT;
1011                 _last_audio_time = time;
1012         } else {
1013                 _last_video_time = optional<DCPTime>();
1014                 _last_video_eyes = optional<Eyes>();
1015                 _last_audio_time = optional<DCPTime>();
1016         }
1017
1018         _black.set_position (time);
1019         _silent.set_position (time);
1020
1021         _last_video.clear ();
1022 }
1023
1024 void
1025 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1026 {
1027         /* We need a delay to give a little wiggle room to ensure that relevent subtitles arrive at the
1028            player before the video that requires them.
1029         */
1030         _delay.push_back (make_pair (pv, time));
1031
1032         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1033                 _last_video_time = time + one_video_frame();
1034         }
1035         _last_video_eyes = increment_eyes (pv->eyes());
1036
1037         if (_delay.size() < 3) {
1038                 return;
1039         }
1040
1041         pair<shared_ptr<PlayerVideo>, DCPTime> to_do = _delay.front();
1042         _delay.pop_front();
1043         do_emit_video (to_do.first, to_do.second);
1044 }
1045
1046 void
1047 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1048 {
1049         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1050                 for (int i = 0; i < CAPTION_COUNT; ++i) {
1051                         _active_captions[i].clear_before (time);
1052                 }
1053         }
1054
1055         optional<PositionImage> captions = open_captions_for_frame (time);
1056         if (captions) {
1057                 pv->set_caption (captions.get ());
1058         }
1059
1060         Video (pv, time);
1061 }
1062
1063 void
1064 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1065 {
1066         /* Log if the assert below is about to fail */
1067         if (_last_audio_time && time != *_last_audio_time) {
1068                 _film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_last_audio_time)), LogEntry::TYPE_WARNING);
1069         }
1070
1071         /* This audio must follow on from the previous */
1072         DCPOMATIC_ASSERT (!_last_audio_time || time == *_last_audio_time);
1073         Audio (data, time);
1074         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate());
1075 }
1076
1077 void
1078 Player::fill_audio (DCPTimePeriod period)
1079 {
1080         if (period.from == period.to) {
1081                 return;
1082         }
1083
1084         DCPOMATIC_ASSERT (period.from < period.to);
1085
1086         DCPTime t = period.from;
1087         while (t < period.to) {
1088                 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1089                 Frame const samples = block.frames_round(_film->audio_frame_rate());
1090                 if (samples) {
1091                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1092                         silence->make_silent ();
1093                         emit_audio (silence, t);
1094                 }
1095                 t += block;
1096         }
1097 }
1098
1099 DCPTime
1100 Player::one_video_frame () const
1101 {
1102         return DCPTime::from_frames (1, _film->video_frame_rate ());
1103 }
1104
1105 pair<shared_ptr<AudioBuffers>, DCPTime>
1106 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1107 {
1108         DCPTime const discard_time = discard_to - time;
1109         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1110         Frame remaining_frames = audio->frames() - discard_frames;
1111         if (remaining_frames <= 0) {
1112                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1113         }
1114         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1115         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1116         return make_pair(cut, time + discard_time);
1117 }
1118
1119 void
1120 Player::set_dcp_decode_reduction (optional<int> reduction)
1121 {
1122         if (reduction == _dcp_decode_reduction) {
1123                 return;
1124         }
1125
1126         _dcp_decode_reduction = reduction;
1127         _have_valid_pieces = false;
1128         Changed (PlayerProperty::DCP_DECODE_REDUCTION, false);
1129 }
1130
1131 DCPTime
1132 Player::content_time_to_dcp (shared_ptr<Content> content, ContentTime t)
1133 {
1134         if (_have_valid_pieces) {
1135                 setup_pieces ();
1136         }
1137
1138         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1139                 if (i->content == content) {
1140                         return content_time_to_dcp (i, t);
1141                 }
1142         }
1143
1144         DCPOMATIC_ASSERT (false);
1145         return DCPTime ();
1146 }