src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2018 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_text.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "caption_content.h"
  44 #include "caption_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "dcp_decoder.h"
  48 #include "image_decoder.h"
  49 #include "compose.hpp"
  50 #include "shuffler.h"
  51 #include <dcp/reel.h>
  52 #include <dcp/reel_sound_asset.h>
  53 #include <dcp/reel_subtitle_asset.h>
  54 #include <dcp/reel_picture_asset.h>
  55 #include <dcp/reel_closed_caption_asset.h>
  56 #include <boost/foreach.hpp>
  57 #include <stdint.h>
  58 #include <algorithm>
  59 #include <iostream>
  60
  61 #include "i18n.h"
  62
  63 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
  64
  65 using std::list;
  66 using std::cout;
  67 using std::min;
  68 using std::max;
  69 using std::min;
  70 using std::vector;
  71 using std::pair;
  72 using std::map;
  73 using std::make_pair;
  74 using std::copy;
  75 using boost::shared_ptr;
  76 using boost::weak_ptr;
  77 using boost::dynamic_pointer_cast;
  78 using boost::optional;
  79 using boost::scoped_ptr;
  80
  81 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
  82 int const PlayerProperty::PLAYLIST = 701;
  83 int const PlayerProperty::FILM_CONTAINER = 702;
  84 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
  85 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
  86
  87 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  88         : _film (film)
  89         , _playlist (playlist)
  90         , _have_valid_pieces (false)
  91         , _ignore_video (false)
  92         , _ignore_caption (false)
  93         , _fast (false)
  94         , _play_referenced (false)
  95         , _audio_merger (_film->audio_frame_rate())
  96         , _shuffler (0)
  97 {
  98         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  99         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 100         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
 101         set_video_container_size (_film->frame_size ());
 102
 103         film_changed (Film::AUDIO_PROCESSOR);
 104
 105         seek (DCPTime (), true);
 106 }
 107
 108 Player::~Player ()
 109 {
 110         delete _shuffler;
 111 }
 112
 113 void
 114 Player::setup_pieces ()
 115 {
 116         _pieces.clear ();
 117
 118         delete _shuffler;
 119         _shuffler = new Shuffler();
 120         _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
 121
 122         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 123
 124                 if (!i->paths_valid ()) {
 125                         continue;
 126                 }
 127
 128                 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log(), _fast);
 129                 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
 130
 131                 if (!decoder) {
 132                         /* Not something that we can decode; e.g. Atmos content */
 133                         continue;
 134                 }
 135
 136                 if (decoder->video && _ignore_video) {
 137                         decoder->video->set_ignore (true);
 138                 }
 139
 140                 if (_ignore_caption) {
 141                         BOOST_FOREACH (shared_ptr<CaptionDecoder> i, decoder->caption) {
 142                                 i->set_ignore (true);
 143                         }
 144                 }
 145
 146                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 147                 if (dcp) {
 148                         dcp->set_decode_referenced (_play_referenced);
 149                         if (_play_referenced) {
 150                                 dcp->set_forced_reduction (_dcp_decode_reduction);
 151                         }
 152                 }
 153
 154                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 155                 _pieces.push_back (piece);
 156
 157                 if (decoder->video) {
 158                         if (i->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT || i->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) {
 159                                 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
 160                                 decoder->video->Data.connect (bind (&Shuffler::video, _shuffler, weak_ptr<Piece>(piece), _1));
 161                         } else {
 162                                 decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece>(piece), _1));
 163                         }
 164                 }
 165
 166                 if (decoder->audio) {
 167                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 168                 }
 169
 170                 list<shared_ptr<CaptionDecoder> >::const_iterator j = decoder->caption.begin();
 171
 172                 while (j != decoder->caption.end()) {
 173                         (*j)->BitmapStart.connect (
 174                                 bind(&Player::bitmap_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const CaptionContent>((*j)->content()), _1)
 175                                 );
 176                         (*j)->PlainStart.connect (
 177                                 bind(&Player::plain_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const CaptionContent>((*j)->content()), _1)
 178                                 );
 179                         (*j)->Stop.connect (
 180                                 bind(&Player::subtitle_stop, this, weak_ptr<Piece>(piece), weak_ptr<const CaptionContent>((*j)->content()), _1, _2)
 181                                 );
 182
 183                         ++j;
 184                 }
 185         }
 186
 187         _stream_states.clear ();
 188         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 189                 if (i->content->audio) {
 190                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 191                                 _stream_states[j] = StreamState (i, i->content->position ());
 192                         }
 193                 }
 194         }
 195
 196         _black = Empty (_film->content(), _film->length(), bind(&Content::video, _1));
 197         _silent = Empty (_film->content(), _film->length(), bind(&Content::audio, _1));
 198
 199         _last_video_time = DCPTime ();
 200         _last_video_eyes = EYES_BOTH;
 201         _last_audio_time = DCPTime ();
 202         _have_valid_pieces = true;
 203 }
 204
 205 void
 206 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
 207 {
 208         shared_ptr<Content> c = w.lock ();
 209         if (!c) {
 210                 return;
 211         }
 212
 213         if (
 214                 property == ContentProperty::POSITION ||
 215                 property == ContentProperty::LENGTH ||
 216                 property == ContentProperty::TRIM_START ||
 217                 property == ContentProperty::TRIM_END ||
 218                 property == ContentProperty::PATH ||
 219                 property == VideoContentProperty::FRAME_TYPE ||
 220                 property == VideoContentProperty::COLOUR_CONVERSION ||
 221                 property == AudioContentProperty::STREAMS ||
 222                 property == DCPContentProperty::NEEDS_ASSETS ||
 223                 property == DCPContentProperty::NEEDS_KDM ||
 224                 property == CaptionContentProperty::COLOUR ||
 225                 property == CaptionContentProperty::EFFECT ||
 226                 property == CaptionContentProperty::EFFECT_COLOUR ||
 227                 property == FFmpegContentProperty::SUBTITLE_STREAM ||
 228                 property == FFmpegContentProperty::FILTERS
 229                 ) {
 230
 231                 _have_valid_pieces = false;
 232                 Changed (property, frequent);
 233
 234         } else if (
 235                 property == CaptionContentProperty::LINE_SPACING ||
 236                 property == CaptionContentProperty::OUTLINE_WIDTH ||
 237                 property == CaptionContentProperty::Y_SCALE ||
 238                 property == CaptionContentProperty::FADE_IN ||
 239                 property == CaptionContentProperty::FADE_OUT ||
 240                 property == ContentProperty::VIDEO_FRAME_RATE ||
 241                 property == CaptionContentProperty::USE ||
 242                 property == CaptionContentProperty::X_OFFSET ||
 243                 property == CaptionContentProperty::Y_OFFSET ||
 244                 property == CaptionContentProperty::X_SCALE ||
 245                 property == CaptionContentProperty::FONTS ||
 246                 property == CaptionContentProperty::TYPE ||
 247                 property == VideoContentProperty::CROP ||
 248                 property == VideoContentProperty::SCALE ||
 249                 property == VideoContentProperty::FADE_IN ||
 250                 property == VideoContentProperty::FADE_OUT
 251                 ) {
 252
 253                 Changed (property, frequent);
 254         }
 255 }
 256
 257 void
 258 Player::set_video_container_size (dcp::Size s)
 259 {
 260         if (s == _video_container_size) {
 261                 return;
 262         }
 263
 264         _video_container_size = s;
 265
 266         _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 267         _black_image->make_black ();
 268
 269         Changed (PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 270 }
 271
 272 void
 273 Player::playlist_changed ()
 274 {
 275         _have_valid_pieces = false;
 276         Changed (PlayerProperty::PLAYLIST, false);
 277 }
 278
 279 void
 280 Player::film_changed (Film::Property p)
 281 {
 282         /* Here we should notice Film properties that affect our output, and
 283            alert listeners that our output now would be different to how it was
 284            last time we were run.
 285         */
 286
 287         if (p == Film::CONTAINER) {
 288                 Changed (PlayerProperty::FILM_CONTAINER, false);
 289         } else if (p == Film::VIDEO_FRAME_RATE) {
 290                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 291                    so we need new pieces here.
 292                 */
 293                 _have_valid_pieces = false;
 294                 Changed (PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
 295         } else if (p == Film::AUDIO_PROCESSOR) {
 296                 if (_film->audio_processor ()) {
 297                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 298                 }
 299         } else if (p == Film::AUDIO_CHANNELS) {
 300                 _audio_merger.clear ();
 301         }
 302 }
 303
 304 list<PositionImage>
 305 Player::transform_bitmap_captions (list<BitmapCaption> subs) const
 306 {
 307         list<PositionImage> all;
 308
 309         for (list<BitmapCaption>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 310                 if (!i->image) {
 311                         continue;
 312                 }
 313
 314                 /* We will scale the subtitle up to fit _video_container_size */
 315                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
 316
 317                 all.push_back (
 318                         PositionImage (
 319                                 i->image->scale (
 320                                         scaled_size,
 321                                         dcp::YUV_TO_RGB_REC601,
 322                                         i->image->pixel_format (),
 323                                         true,
 324                                         _fast
 325                                         ),
 326                                 Position<int> (
 327                                         lrint (_video_container_size.width * i->rectangle.x),
 328                                         lrint (_video_container_size.height * i->rectangle.y)
 329                                         )
 330                                 )
 331                         );
 332         }
 333
 334         return all;
 335 }
 336
 337 shared_ptr<PlayerVideo>
 338 Player::black_player_video_frame (Eyes eyes) const
 339 {
 340         return shared_ptr<PlayerVideo> (
 341                 new PlayerVideo (
 342                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 343                         Crop (),
 344                         optional<double> (),
 345                         _video_container_size,
 346                         _video_container_size,
 347                         eyes,
 348                         PART_WHOLE,
 349                         PresetColourConversion::all().front().conversion,
 350                         boost::weak_ptr<Content>(),
 351                         boost::optional<Frame>()
 352                 )
 353         );
 354 }
 355
 356 Frame
 357 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 358 {
 359         DCPTime s = t - piece->content->position ();
 360         s = min (piece->content->length_after_trim(), s);
 361         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 362
 363         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 364            then convert that ContentTime to frames at the content's rate.  However this fails for
 365            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 366            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 367
 368            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 369         */
 370         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 371 }
 372
 373 DCPTime
 374 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 375 {
 376         /* See comment in dcp_to_content_video */
 377         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime(piece->content->trim_start(), piece->frc);
 378         return d + piece->content->position();
 379 }
 380
 381 Frame
 382 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 383 {
 384         DCPTime s = t - piece->content->position ();
 385         s = min (piece->content->length_after_trim(), s);
 386         /* See notes in dcp_to_content_video */
 387         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 388 }
 389
 390 DCPTime
 391 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 392 {
 393         /* See comment in dcp_to_content_video */
 394         return DCPTime::from_frames (f, _film->audio_frame_rate())
 395                 - DCPTime (piece->content->trim_start(), piece->frc)
 396                 + piece->content->position();
 397 }
 398
 399 ContentTime
 400 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 401 {
 402         DCPTime s = t - piece->content->position ();
 403         s = min (piece->content->length_after_trim(), s);
 404         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 405 }
 406
 407 DCPTime
 408 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 409 {
 410         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 411 }
 412
 413 list<shared_ptr<Font> >
 414 Player::get_subtitle_fonts ()
 415 {
 416         if (!_have_valid_pieces) {
 417                 setup_pieces ();
 418         }
 419
 420         list<shared_ptr<Font> > fonts;
 421         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 422                 BOOST_FOREACH (shared_ptr<CaptionContent> j, i->content->caption) {
 423                         /* XXX: things may go wrong if there are duplicate font IDs
 424                            with different font files.
 425                         */
 426                         list<shared_ptr<Font> > f = j->fonts ();
 427                         copy (f.begin(), f.end(), back_inserter (fonts));
 428                 }
 429         }
 430
 431         return fonts;
 432 }
 433
 434 /** Set this player never to produce any video data */
 435 void
 436 Player::set_ignore_video ()
 437 {
 438         _ignore_video = true;
 439 }
 440
 441 void
 442 Player::set_ignore_caption ()
 443 {
 444         _ignore_caption = true;
 445 }
 446
 447 /** Set the player to always burn open captions into the image regardless of the content settings */
 448 void
 449 Player::set_always_burn_open_captions ()
 450 {
 451         _always_burn_open_captions = true;
 452 }
 453
 454 /** Sets up the player to be faster, possibly at the expense of quality */
 455 void
 456 Player::set_fast ()
 457 {
 458         _fast = true;
 459         _have_valid_pieces = false;
 460 }
 461
 462 void
 463 Player::set_play_referenced ()
 464 {
 465         _play_referenced = true;
 466         _have_valid_pieces = false;
 467 }
 468
 469 list<ReferencedReelAsset>
 470 Player::get_reel_assets ()
 471 {
 472         list<ReferencedReelAsset> a;
 473
 474         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 475                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 476                 if (!j) {
 477                         continue;
 478                 }
 479
 480                 scoped_ptr<DCPDecoder> decoder;
 481                 try {
 482                         decoder.reset (new DCPDecoder (j, _film->log(), false));
 483                 } catch (...) {
 484                         return a;
 485                 }
 486
 487                 int64_t offset = 0;
 488                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 489
 490                         DCPOMATIC_ASSERT (j->video_frame_rate ());
 491                         double const cfr = j->video_frame_rate().get();
 492                         Frame const trim_start = j->trim_start().frames_round (cfr);
 493                         Frame const trim_end = j->trim_end().frames_round (cfr);
 494                         int const ffr = _film->video_frame_rate ();
 495
 496                         DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
 497                         if (j->reference_video ()) {
 498                                 shared_ptr<dcp::ReelAsset> ra = k->main_picture ();
 499                                 DCPOMATIC_ASSERT (ra);
 500                                 ra->set_entry_point (ra->entry_point() + trim_start);
 501                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 502                                 a.push_back (
 503                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 504                                         );
 505                         }
 506
 507                         if (j->reference_audio ()) {
 508                                 shared_ptr<dcp::ReelAsset> ra = k->main_sound ();
 509                                 DCPOMATIC_ASSERT (ra);
 510                                 ra->set_entry_point (ra->entry_point() + trim_start);
 511                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 512                                 a.push_back (
 513                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 514                                         );
 515                         }
 516
 517                         if (j->reference_caption (CAPTION_OPEN)) {
 518                                 shared_ptr<dcp::ReelAsset> ra = k->main_subtitle ();
 519                                 DCPOMATIC_ASSERT (ra);
 520                                 ra->set_entry_point (ra->entry_point() + trim_start);
 521                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 522                                 a.push_back (
 523                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 524                                         );
 525                         }
 526
 527                         if (j->reference_caption (CAPTION_CLOSED)) {
 528                                 shared_ptr<dcp::ReelAsset> ra = k->closed_caption ();
 529                                 DCPOMATIC_ASSERT (ra);
 530                                 ra->set_entry_point (ra->entry_point() + trim_start);
 531                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 532                                 a.push_back (
 533                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 534                                         );
 535                         }
 536
 537                         /* Assume that main picture duration is the length of the reel */
 538                         offset += k->main_picture()->duration ();
 539                 }
 540         }
 541
 542         return a;
 543 }
 544
 545 bool
 546 Player::pass ()
 547 {
 548         if (!_have_valid_pieces) {
 549                 setup_pieces ();
 550         }
 551
 552         if (_playlist->length() == DCPTime()) {
 553                 /* Special case of an empty Film; just give one black frame */
 554                 emit_video (black_player_video_frame(EYES_BOTH), DCPTime());
 555                 return true;
 556         }
 557
 558         /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
 559
 560         shared_ptr<Piece> earliest_content;
 561         optional<DCPTime> earliest_time;
 562
 563         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 564                 if (i->done) {
 565                         continue;
 566                 }
 567
 568                 DCPTime const t = content_time_to_dcp (i, max(i->decoder->position(), i->content->trim_start()));
 569                 if (t > i->content->end()) {
 570                         i->done = true;
 571                 } else {
 572
 573                         /* Given two choices at the same time, pick the one with captions so we see it before
 574                            the video.
 575                         */
 576                         if (!earliest_time || t < *earliest_time || (t == *earliest_time && !i->decoder->caption.empty())) {
 577                                 earliest_time = t;
 578                                 earliest_content = i;
 579                         }
 580                 }
 581         }
 582
 583         bool done = false;
 584
 585         enum {
 586                 NONE,
 587                 CONTENT,
 588                 BLACK,
 589                 SILENT
 590         } which = NONE;
 591
 592         if (earliest_content) {
 593                 which = CONTENT;
 594         }
 595
 596         if (!_black.done() && (!earliest_time || _black.position() < *earliest_time)) {
 597                 earliest_time = _black.position ();
 598                 which = BLACK;
 599         }
 600
 601         if (!_silent.done() && (!earliest_time || _silent.position() < *earliest_time)) {
 602                 earliest_time = _silent.position ();
 603                 which = SILENT;
 604         }
 605
 606         switch (which) {
 607         case CONTENT:
 608                 earliest_content->done = earliest_content->decoder->pass ();
 609                 break;
 610         case BLACK:
 611                 emit_video (black_player_video_frame(EYES_BOTH), _black.position());
 612                 _black.set_position (_black.position() + one_video_frame());
 613                 break;
 614         case SILENT:
 615         {
 616                 DCPTimePeriod period (_silent.period_at_position());
 617                 if (_last_audio_time) {
 618                         /* Sometimes the thing that happened last finishes fractionally before
 619                            this silence.  Bodge the start time of the silence to fix it.  I'm
 620                            not sure if this is the right solution --- maybe the last thing should
 621                            be padded `forward' rather than this thing padding `back'.
 622                         */
 623                         period.from = min(period.from, *_last_audio_time);
 624                 }
 625                 if (period.duration() > one_video_frame()) {
 626                         period.to = period.from + one_video_frame();
 627                 }
 628                 fill_audio (period);
 629                 _silent.set_position (period.to);
 630                 break;
 631         }
 632         case NONE:
 633                 done = true;
 634                 break;
 635         }
 636
 637         /* Emit any audio that is ready */
 638
 639         /* Work out the time before which the audio is definitely all here.  This is the earliest last_push_end of one
 640            of our streams, or the position of the _silent.
 641         */
 642         DCPTime pull_to = _film->length ();
 643         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 644                 if (!i->second.piece->done && i->second.last_push_end < pull_to) {
 645                         pull_to = i->second.last_push_end;
 646                 }
 647         }
 648         if (!_silent.done() && _silent.position() < pull_to) {
 649                 pull_to = _silent.position();
 650         }
 651
 652         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 653         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 654                 if (_last_audio_time && i->second < *_last_audio_time) {
 655                         /* This new data comes before the last we emitted (or the last seek); discard it */
 656                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 657                         if (!cut.first) {
 658                                 continue;
 659                         }
 660                         *i = cut;
 661                 } else if (_last_audio_time && i->second > *_last_audio_time) {
 662                         /* There's a gap between this data and the last we emitted; fill with silence */
 663                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 664                 }
 665
 666                 emit_audio (i->first, i->second);
 667         }
 668
 669         if (done) {
 670                 _shuffler->flush ();
 671                 for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 672                         do_emit_video(i->first, i->second);
 673                 }
 674         }
 675
 676         return done;
 677 }
 678
 679 optional<PositionImage>
 680 Player::captions_for_frame (DCPTime time) const
 681 {
 682         list<PositionImage> captions;
 683
 684         int const vfr = _film->video_frame_rate();
 685
 686         for (int i = 0; i < CAPTION_COUNT; ++i) {
 687                 bool const always = i == CAPTION_OPEN && _always_burn_open_captions;
 688                 BOOST_FOREACH (
 689                         PlayerCaption j,
 690                         _active_captions[i].get_burnt(DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), always)
 691                         ) {
 692
 693                         /* Image subtitles */
 694                         list<PositionImage> c = transform_bitmap_captions (j.image);
 695                         copy (c.begin(), c.end(), back_inserter (captions));
 696
 697                         /* Text subtitles (rendered to an image) */
 698                         if (!j.text.empty ()) {
 699                                 list<PositionImage> s = render_text (j.text, j.fonts, _video_container_size, time, vfr);
 700                                 copy (s.begin(), s.end(), back_inserter (captions));
 701                         }
 702                 }
 703         }
 704
 705         if (captions.empty ()) {
 706                 return optional<PositionImage> ();
 707         }
 708
 709         return merge (captions);
 710 }
 711
 712 void
 713 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 714 {
 715         shared_ptr<Piece> piece = wp.lock ();
 716         if (!piece) {
 717                 return;
 718         }
 719
 720         FrameRateChange frc(piece->content->active_video_frame_rate(), _film->video_frame_rate());
 721         if (frc.skip && (video.frame % 2) == 1) {
 722                 return;
 723         }
 724
 725         /* Time of the first frame we will emit */
 726         DCPTime const time = content_video_to_dcp (piece, video.frame);
 727
 728         /* Discard if it's before the content's period or the last accurate seek.  We can't discard
 729            if it's after the content's period here as in that case we still need to fill any gap between
 730            `now' and the end of the content's period.
 731         */
 732         if (time < piece->content->position() || (_last_video_time && time < *_last_video_time)) {
 733                 return;
 734         }
 735
 736         /* Fill gaps that we discover now that we have some video which needs to be emitted.
 737            This is where we need to fill to.
 738         */
 739         DCPTime fill_to = min (time, piece->content->end());
 740
 741         if (_last_video_time) {
 742                 DCPTime fill_from = max (*_last_video_time, piece->content->position());
 743                 LastVideoMap::const_iterator last = _last_video.find (wp);
 744                 if (_film->three_d()) {
 745                         DCPTime j = fill_from;
 746                         Eyes eyes = _last_video_eyes.get_value_or(EYES_LEFT);
 747                         if (eyes == EYES_BOTH) {
 748                                 eyes = EYES_LEFT;
 749                         }
 750                         while (j < fill_to || eyes != video.eyes) {
 751                                 if (last != _last_video.end()) {
 752                                         shared_ptr<PlayerVideo> copy = last->second->shallow_copy();
 753                                         copy->set_eyes (eyes);
 754                                         emit_video (copy, j);
 755                                 } else {
 756                                         emit_video (black_player_video_frame(eyes), j);
 757                                 }
 758                                 if (eyes == EYES_RIGHT) {
 759                                         j += one_video_frame();
 760                                 }
 761                                 eyes = increment_eyes (eyes);
 762                         }
 763                 } else {
 764                         for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
 765                                 if (last != _last_video.end()) {
 766                                         emit_video (last->second, j);
 767                                 } else {
 768                                         emit_video (black_player_video_frame(EYES_BOTH), j);
 769                                 }
 770                         }
 771                 }
 772         }
 773
 774         _last_video[wp].reset (
 775                 new PlayerVideo (
 776                         video.image,
 777                         piece->content->video->crop (),
 778                         piece->content->video->fade (video.frame),
 779                         piece->content->video->scale().size (
 780                                 piece->content->video, _video_container_size, _film->frame_size ()
 781                                 ),
 782                         _video_container_size,
 783                         video.eyes,
 784                         video.part,
 785                         piece->content->video->colour_conversion(),
 786                         piece->content,
 787                         video.frame
 788                         )
 789                 );
 790
 791         DCPTime t = time;
 792         for (int i = 0; i < frc.repeat; ++i) {
 793                 if (t < piece->content->end()) {
 794                         emit_video (_last_video[wp], t);
 795                 }
 796                 t += one_video_frame ();
 797         }
 798 }
 799
 800 void
 801 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 802 {
 803         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 804
 805         shared_ptr<Piece> piece = wp.lock ();
 806         if (!piece) {
 807                 return;
 808         }
 809
 810         shared_ptr<AudioContent> content = piece->content->audio;
 811         DCPOMATIC_ASSERT (content);
 812
 813         /* Compute time in the DCP */
 814         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame);
 815         /* And the end of this block in the DCP */
 816         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), content->resampled_frame_rate());
 817
 818         /* Remove anything that comes before the start or after the end of the content */
 819         if (time < piece->content->position()) {
 820                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 821                 if (!cut.first) {
 822                         /* This audio is entirely discarded */
 823                         return;
 824                 }
 825                 content_audio.audio = cut.first;
 826                 time = cut.second;
 827         } else if (time > piece->content->end()) {
 828                 /* Discard it all */
 829                 return;
 830         } else if (end > piece->content->end()) {
 831                 Frame const remaining_frames = DCPTime(piece->content->end() - time).frames_round(_film->audio_frame_rate());
 832                 if (remaining_frames == 0) {
 833                         return;
 834                 }
 835                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 836                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 837                 content_audio.audio = cut;
 838         }
 839
 840         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 841
 842         /* Gain */
 843
 844         if (content->gain() != 0) {
 845                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 846                 gain->apply_gain (content->gain ());
 847                 content_audio.audio = gain;
 848         }
 849
 850         /* Remap */
 851
 852         content_audio.audio = remap (content_audio.audio, _film->audio_channels(), stream->mapping());
 853
 854         /* Process */
 855
 856         if (_audio_processor) {
 857                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 858         }
 859
 860         /* Push */
 861
 862         _audio_merger.push (content_audio.audio, time);
 863         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 864         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 865 }
 866
 867 void
 868 Player::bitmap_text_start (weak_ptr<Piece> wp, weak_ptr<const CaptionContent> wc, ContentBitmapCaption subtitle)
 869 {
 870         shared_ptr<Piece> piece = wp.lock ();
 871         shared_ptr<const CaptionContent> caption = wc.lock ();
 872         if (!piece || !caption) {
 873                 return;
 874         }
 875
 876         /* Apply content's subtitle offsets */
 877         subtitle.sub.rectangle.x += caption->x_offset ();
 878         subtitle.sub.rectangle.y += caption->y_offset ();
 879
 880         /* Apply a corrective translation to keep the subtitle centred after the scale that is coming up */
 881         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * ((caption->x_scale() - 1) / 2);
 882         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * ((caption->y_scale() - 1) / 2);
 883
 884         /* Apply content's subtitle scale */
 885         subtitle.sub.rectangle.width *= caption->x_scale ();
 886         subtitle.sub.rectangle.height *= caption->y_scale ();
 887
 888         PlayerCaption ps;
 889         ps.image.push_back (subtitle.sub);
 890         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 891
 892         _active_captions[subtitle.type()].add_from (wc, ps, from);
 893 }
 894
 895 void
 896 Player::plain_text_start (weak_ptr<Piece> wp, weak_ptr<const CaptionContent> wc, ContentTextCaption subtitle)
 897 {
 898         shared_ptr<Piece> piece = wp.lock ();
 899         shared_ptr<const CaptionContent> caption = wc.lock ();
 900         if (!piece || !caption) {
 901                 return;
 902         }
 903
 904         PlayerCaption ps;
 905         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 906
 907         if (from > piece->content->end()) {
 908                 return;
 909         }
 910
 911         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 912                 s.set_h_position (s.h_position() + caption->x_offset ());
 913                 s.set_v_position (s.v_position() + caption->y_offset ());
 914                 float const xs = caption->x_scale();
 915                 float const ys = caption->y_scale();
 916                 float size = s.size();
 917
 918                 /* Adjust size to express the common part of the scaling;
 919                    e.g. if xs = ys = 0.5 we scale size by 2.
 920                 */
 921                 if (xs > 1e-5 && ys > 1e-5) {
 922                         size *= 1 / min (1 / xs, 1 / ys);
 923                 }
 924                 s.set_size (size);
 925
 926                 /* Then express aspect ratio changes */
 927                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 928                         s.set_aspect_adjust (xs / ys);
 929                 }
 930
 931                 s.set_in (dcp::Time(from.seconds(), 1000));
 932                 ps.text.push_back (TextCaption (s, caption->outline_width()));
 933                 ps.add_fonts (caption->fonts ());
 934         }
 935
 936         _active_captions[subtitle.type()].add_from (wc, ps, from);
 937 }
 938
 939 void
 940 Player::subtitle_stop (weak_ptr<Piece> wp, weak_ptr<const CaptionContent> wc, ContentTime to, CaptionType type)
 941 {
 942         if (!_active_captions[type].have (wc)) {
 943                 return;
 944         }
 945
 946         shared_ptr<Piece> piece = wp.lock ();
 947         shared_ptr<const CaptionContent> caption = wc.lock ();
 948         if (!piece || !caption) {
 949                 return;
 950         }
 951
 952         DCPTime const dcp_to = content_time_to_dcp (piece, to);
 953
 954         if (dcp_to > piece->content->end()) {
 955                 return;
 956         }
 957
 958         pair<PlayerCaption, DCPTime> from = _active_captions[type].add_to (wc, dcp_to);
 959
 960         bool const always = type == CAPTION_OPEN && _always_burn_open_captions;
 961         if (caption->use() && !always && !caption->burn()) {
 962                 Caption (from.first, type, DCPTimePeriod (from.second, dcp_to));
 963         }
 964 }
 965
 966 void
 967 Player::seek (DCPTime time, bool accurate)
 968 {
 969         if (!_have_valid_pieces) {
 970                 setup_pieces ();
 971         }
 972
 973         if (_shuffler) {
 974                 _shuffler->clear ();
 975         }
 976
 977         _delay.clear ();
 978
 979         if (_audio_processor) {
 980                 _audio_processor->flush ();
 981         }
 982
 983         _audio_merger.clear ();
 984         for (int i = 0; i < CAPTION_COUNT; ++i) {
 985                 _active_captions[i].clear ();
 986         }
 987
 988         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 989                 if (time < i->content->position()) {
 990                         /* Before; seek to the start of the content */
 991                         i->decoder->seek (dcp_to_content_time (i, i->content->position()), accurate);
 992                         i->done = false;
 993                 } else if (i->content->position() <= time && time < i->content->end()) {
 994                         /* During; seek to position */
 995                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
 996                         i->done = false;
 997                 } else {
 998                         /* After; this piece is done */
 999                         i->done = true;
1000                 }
1001         }
1002
1003         if (accurate) {
1004                 _last_video_time = time;
1005                 _last_video_eyes = EYES_LEFT;
1006                 _last_audio_time = time;
1007         } else {
1008                 _last_video_time = optional<DCPTime>();
1009                 _last_video_eyes = optional<Eyes>();
1010                 _last_audio_time = optional<DCPTime>();
1011         }
1012
1013         _black.set_position (time);
1014         _silent.set_position (time);
1015
1016         _last_video.clear ();
1017 }
1018
1019 void
1020 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1021 {
1022         /* We need a delay to give a little wiggle room to ensure that relevent subtitles arrive at the
1023            player before the video that requires them.
1024         */
1025         _delay.push_back (make_pair (pv, time));
1026
1027         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1028                 _last_video_time = time + one_video_frame();
1029         }
1030         _last_video_eyes = increment_eyes (pv->eyes());
1031
1032         if (_delay.size() < 3) {
1033                 return;
1034         }
1035
1036         pair<shared_ptr<PlayerVideo>, DCPTime> to_do = _delay.front();
1037         _delay.pop_front();
1038         do_emit_video (to_do.first, to_do.second);
1039 }
1040
1041 void
1042 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1043 {
1044         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1045                 for (int i = 0; i < CAPTION_COUNT; ++i) {
1046                         _active_captions[i].clear_before (time);
1047                 }
1048         }
1049
1050         optional<PositionImage> captions = captions_for_frame (time);
1051         if (captions) {
1052                 pv->set_caption (captions.get ());
1053         }
1054
1055         Video (pv, time);
1056 }
1057
1058 void
1059 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1060 {
1061         /* Log if the assert below is about to fail */
1062         if (_last_audio_time && time != *_last_audio_time) {
1063                 _film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_last_audio_time)), LogEntry::TYPE_WARNING);
1064         }
1065
1066         /* This audio must follow on from the previous */
1067         DCPOMATIC_ASSERT (!_last_audio_time || time == *_last_audio_time);
1068         Audio (data, time);
1069         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate());
1070 }
1071
1072 void
1073 Player::fill_audio (DCPTimePeriod period)
1074 {
1075         if (period.from == period.to) {
1076                 return;
1077         }
1078
1079         DCPOMATIC_ASSERT (period.from < period.to);
1080
1081         DCPTime t = period.from;
1082         while (t < period.to) {
1083                 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1084                 Frame const samples = block.frames_round(_film->audio_frame_rate());
1085                 if (samples) {
1086                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1087                         silence->make_silent ();
1088                         emit_audio (silence, t);
1089                 }
1090                 t += block;
1091         }
1092 }
1093
1094 DCPTime
1095 Player::one_video_frame () const
1096 {
1097         return DCPTime::from_frames (1, _film->video_frame_rate ());
1098 }
1099
1100 pair<shared_ptr<AudioBuffers>, DCPTime>
1101 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1102 {
1103         DCPTime const discard_time = discard_to - time;
1104         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1105         Frame remaining_frames = audio->frames() - discard_frames;
1106         if (remaining_frames <= 0) {
1107                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1108         }
1109         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1110         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1111         return make_pair(cut, time + discard_time);
1112 }
1113
1114 void
1115 Player::set_dcp_decode_reduction (optional<int> reduction)
1116 {
1117         if (reduction == _dcp_decode_reduction) {
1118                 return;
1119         }
1120
1121         _dcp_decode_reduction = reduction;
1122         _have_valid_pieces = false;
1123         Changed (PlayerProperty::DCP_DECODE_REDUCTION, false);
1124 }
1125
1126 DCPTime
1127 Player::content_time_to_dcp (shared_ptr<Content> content, ContentTime t)
1128 {
1129         if (_have_valid_pieces) {
1130                 setup_pieces ();
1131         }
1132
1133         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1134                 if (i->content == content) {
1135                         return content_time_to_dcp (i, t);
1136                 }
1137         }
1138
1139         DCPOMATIC_ASSERT (false);
1140         return DCPTime ();
1141 }