src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2018 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_subtitles.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "subtitle_content.h"
  44 #include "subtitle_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "content_subtitle.h"
  48 #include "dcp_decoder.h"
  49 #include "image_decoder.h"
  50 #include "compose.hpp"
  51 #include "shuffler.h"
  52 #include <dcp/reel.h>
  53 #include <dcp/reel_sound_asset.h>
  54 #include <dcp/reel_subtitle_asset.h>
  55 #include <dcp/reel_picture_asset.h>
  56 #include <boost/foreach.hpp>
  57 #include <stdint.h>
  58 #include <algorithm>
  59 #include <iostream>
  60
  61 #include "i18n.h"
  62
  63 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
  64
  65 using std::list;
  66 using std::cout;
  67 using std::min;
  68 using std::max;
  69 using std::min;
  70 using std::vector;
  71 using std::pair;
  72 using std::map;
  73 using std::make_pair;
  74 using std::copy;
  75 using boost::shared_ptr;
  76 using boost::weak_ptr;
  77 using boost::dynamic_pointer_cast;
  78 using boost::optional;
  79 using boost::scoped_ptr;
  80
  81 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
  82 int const PlayerProperty::PLAYLIST = 701;
  83 int const PlayerProperty::FILM_CONTAINER = 702;
  84 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
  85 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
  86
  87 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  88         : _film (film)
  89         , _playlist (playlist)
  90         , _have_valid_pieces (false)
  91         , _ignore_video (false)
  92         , _ignore_subtitle (false)
  93         , _always_burn_subtitles (false)
  94         , _fast (false)
  95         , _play_referenced (false)
  96         , _audio_merger (_film->audio_frame_rate())
  97         , _shuffler (0)
  98 {
  99         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
 100         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 101         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
 102         set_video_container_size (_film->frame_size ());
 103
 104         film_changed (Film::AUDIO_PROCESSOR);
 105
 106         seek (DCPTime (), true);
 107 }
 108
 109 Player::~Player ()
 110 {
 111         delete _shuffler;
 112 }
 113
 114 void
 115 Player::setup_pieces ()
 116 {
 117         _pieces.clear ();
 118
 119         delete _shuffler;
 120         _shuffler = new Shuffler();
 121         _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
 122
 123         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 124
 125                 if (!i->paths_valid ()) {
 126                         continue;
 127                 }
 128
 129                 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log(), _fast);
 130                 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
 131
 132                 if (!decoder) {
 133                         /* Not something that we can decode; e.g. Atmos content */
 134                         continue;
 135                 }
 136
 137                 if (decoder->video && _ignore_video) {
 138                         decoder->video->set_ignore (true);
 139                 }
 140
 141                 if (decoder->subtitle && _ignore_subtitle) {
 142                         decoder->subtitle->set_ignore (true);
 143                 }
 144
 145                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 146                 if (dcp) {
 147                         dcp->set_decode_referenced (_play_referenced);
 148                         if (_play_referenced) {
 149                                 dcp->set_forced_reduction (_dcp_decode_reduction);
 150                         }
 151                 }
 152
 153                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 154                 _pieces.push_back (piece);
 155
 156                 if (decoder->video) {
 157                         if (i->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT || i->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) {
 158                                 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
 159                                 decoder->video->Data.connect (bind (&Shuffler::video, _shuffler, weak_ptr<Piece>(piece), _1));
 160                         } else {
 161                                 decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece>(piece), _1));
 162                         }
 163                 }
 164
 165                 if (decoder->audio) {
 166                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 167                 }
 168
 169                 if (decoder->subtitle) {
 170                         decoder->subtitle->ImageStart.connect (bind (&Player::image_subtitle_start, this, weak_ptr<Piece> (piece), _1));
 171                         decoder->subtitle->TextStart.connect (bind (&Player::text_subtitle_start, this, weak_ptr<Piece> (piece), _1));
 172                         decoder->subtitle->Stop.connect (bind (&Player::subtitle_stop, this, weak_ptr<Piece> (piece), _1));
 173                 }
 174         }
 175
 176         _stream_states.clear ();
 177         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 178                 if (i->content->audio) {
 179                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 180                                 _stream_states[j] = StreamState (i, i->content->position ());
 181                         }
 182                 }
 183         }
 184
 185         _black = Empty (_film->content(), _film->length(), bind(&Content::video, _1));
 186         _silent = Empty (_film->content(), _film->length(), bind(&Content::audio, _1));
 187
 188         _last_video_time = DCPTime ();
 189         _last_video_eyes = EYES_BOTH;
 190         _last_audio_time = DCPTime ();
 191         _have_valid_pieces = true;
 192 }
 193
 194 void
 195 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
 196 {
 197         shared_ptr<Content> c = w.lock ();
 198         if (!c) {
 199                 return;
 200         }
 201
 202         if (
 203                 property == ContentProperty::POSITION ||
 204                 property == ContentProperty::LENGTH ||
 205                 property == ContentProperty::TRIM_START ||
 206                 property == ContentProperty::TRIM_END ||
 207                 property == ContentProperty::PATH ||
 208                 property == VideoContentProperty::FRAME_TYPE ||
 209                 property == DCPContentProperty::NEEDS_ASSETS ||
 210                 property == DCPContentProperty::NEEDS_KDM ||
 211                 property == SubtitleContentProperty::COLOUR ||
 212                 property == SubtitleContentProperty::EFFECT ||
 213                 property == SubtitleContentProperty::EFFECT_COLOUR ||
 214                 property == FFmpegContentProperty::SUBTITLE_STREAM ||
 215                 property == FFmpegContentProperty::FILTERS ||
 216                 property == VideoContentProperty::COLOUR_CONVERSION
 217                 ) {
 218
 219                 _have_valid_pieces = false;
 220                 Changed (property, frequent);
 221
 222         } else if (
 223                 property == SubtitleContentProperty::LINE_SPACING ||
 224                 property == SubtitleContentProperty::OUTLINE_WIDTH ||
 225                 property == SubtitleContentProperty::Y_SCALE ||
 226                 property == SubtitleContentProperty::FADE_IN ||
 227                 property == SubtitleContentProperty::FADE_OUT ||
 228                 property == ContentProperty::VIDEO_FRAME_RATE ||
 229                 property == SubtitleContentProperty::USE ||
 230                 property == SubtitleContentProperty::X_OFFSET ||
 231                 property == SubtitleContentProperty::Y_OFFSET ||
 232                 property == SubtitleContentProperty::X_SCALE ||
 233                 property == SubtitleContentProperty::FONTS ||
 234                 property == VideoContentProperty::CROP ||
 235                 property == VideoContentProperty::SCALE ||
 236                 property == VideoContentProperty::FADE_IN ||
 237                 property == VideoContentProperty::FADE_OUT
 238                 ) {
 239
 240                 Changed (property, frequent);
 241         }
 242 }
 243
 244 void
 245 Player::set_video_container_size (dcp::Size s)
 246 {
 247         if (s == _video_container_size) {
 248                 return;
 249         }
 250
 251         _video_container_size = s;
 252
 253         _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 254         _black_image->make_black ();
 255
 256         Changed (PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 257 }
 258
 259 void
 260 Player::playlist_changed ()
 261 {
 262         _have_valid_pieces = false;
 263         Changed (PlayerProperty::PLAYLIST, false);
 264 }
 265
 266 void
 267 Player::film_changed (Film::Property p)
 268 {
 269         /* Here we should notice Film properties that affect our output, and
 270            alert listeners that our output now would be different to how it was
 271            last time we were run.
 272         */
 273
 274         if (p == Film::CONTAINER) {
 275                 Changed (PlayerProperty::FILM_CONTAINER, false);
 276         } else if (p == Film::VIDEO_FRAME_RATE) {
 277                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 278                    so we need new pieces here.
 279                 */
 280                 _have_valid_pieces = false;
 281                 Changed (PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
 282         } else if (p == Film::AUDIO_PROCESSOR) {
 283                 if (_film->audio_processor ()) {
 284                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 285                 }
 286         } else if (p == Film::AUDIO_CHANNELS) {
 287                 _audio_merger.clear ();
 288         }
 289 }
 290
 291 list<PositionImage>
 292 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
 293 {
 294         list<PositionImage> all;
 295
 296         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 297                 if (!i->image) {
 298                         continue;
 299                 }
 300
 301                 /* We will scale the subtitle up to fit _video_container_size */
 302                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
 303
 304                 /* Then we need a corrective translation, consisting of two parts:
 305                  *
 306                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 307                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 308                  *
 309                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 310                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
 311                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
 312                  *
 313                  * Combining these two translations gives these expressions.
 314                  */
 315
 316                 all.push_back (
 317                         PositionImage (
 318                                 i->image->scale (
 319                                         scaled_size,
 320                                         dcp::YUV_TO_RGB_REC601,
 321                                         i->image->pixel_format (),
 322                                         true,
 323                                         _fast
 324                                         ),
 325                                 Position<int> (
 326                                         lrint (_video_container_size.width * i->rectangle.x),
 327                                         lrint (_video_container_size.height * i->rectangle.y)
 328                                         )
 329                                 )
 330                         );
 331         }
 332
 333         return all;
 334 }
 335
 336 shared_ptr<PlayerVideo>
 337 Player::black_player_video_frame (Eyes eyes) const
 338 {
 339         return shared_ptr<PlayerVideo> (
 340                 new PlayerVideo (
 341                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 342                         Crop (),
 343                         optional<double> (),
 344                         _video_container_size,
 345                         _video_container_size,
 346                         eyes,
 347                         PART_WHOLE,
 348                         PresetColourConversion::all().front().conversion,
 349                         boost::weak_ptr<Content>(),
 350                         boost::optional<Frame>()
 351                 )
 352         );
 353 }
 354
 355 Frame
 356 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 357 {
 358         DCPTime s = t - piece->content->position ();
 359         s = min (piece->content->length_after_trim(), s);
 360         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 361
 362         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 363            then convert that ContentTime to frames at the content's rate.  However this fails for
 364            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 365            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 366
 367            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 368         */
 369         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 370 }
 371
 372 DCPTime
 373 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 374 {
 375         /* See comment in dcp_to_content_video */
 376         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime(piece->content->trim_start(), piece->frc);
 377         return d + piece->content->position();
 378 }
 379
 380 Frame
 381 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 382 {
 383         DCPTime s = t - piece->content->position ();
 384         s = min (piece->content->length_after_trim(), s);
 385         /* See notes in dcp_to_content_video */
 386         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 387 }
 388
 389 DCPTime
 390 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 391 {
 392         /* See comment in dcp_to_content_video */
 393         return DCPTime::from_frames (f, _film->audio_frame_rate())
 394                 - DCPTime (piece->content->trim_start(), piece->frc)
 395                 + piece->content->position();
 396 }
 397
 398 ContentTime
 399 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 400 {
 401         DCPTime s = t - piece->content->position ();
 402         s = min (piece->content->length_after_trim(), s);
 403         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 404 }
 405
 406 DCPTime
 407 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 408 {
 409         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 410 }
 411
 412 list<shared_ptr<Font> >
 413 Player::get_subtitle_fonts ()
 414 {
 415         if (!_have_valid_pieces) {
 416                 setup_pieces ();
 417         }
 418
 419         list<shared_ptr<Font> > fonts;
 420         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
 421                 if (p->content->subtitle) {
 422                         /* XXX: things may go wrong if there are duplicate font IDs
 423                            with different font files.
 424                         */
 425                         list<shared_ptr<Font> > f = p->content->subtitle->fonts ();
 426                         copy (f.begin(), f.end(), back_inserter (fonts));
 427                 }
 428         }
 429
 430         return fonts;
 431 }
 432
 433 /** Set this player never to produce any video data */
 434 void
 435 Player::set_ignore_video ()
 436 {
 437         _ignore_video = true;
 438 }
 439
 440 void
 441 Player::set_ignore_subtitle ()
 442 {
 443         _ignore_subtitle = true;
 444 }
 445
 446 /** Set whether or not this player should always burn text subtitles into the image,
 447  *  regardless of the content settings.
 448  *  @param burn true to always burn subtitles, false to obey content settings.
 449  */
 450 void
 451 Player::set_always_burn_subtitles (bool burn)
 452 {
 453         _always_burn_subtitles = burn;
 454 }
 455
 456 /** Sets up the player to be faster, possibly at the expense of quality */
 457 void
 458 Player::set_fast ()
 459 {
 460         _fast = true;
 461         _have_valid_pieces = false;
 462 }
 463
 464 void
 465 Player::set_play_referenced ()
 466 {
 467         _play_referenced = true;
 468         _have_valid_pieces = false;
 469 }
 470
 471 list<ReferencedReelAsset>
 472 Player::get_reel_assets ()
 473 {
 474         list<ReferencedReelAsset> a;
 475
 476         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 477                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 478                 if (!j) {
 479                         continue;
 480                 }
 481
 482                 scoped_ptr<DCPDecoder> decoder;
 483                 try {
 484                         decoder.reset (new DCPDecoder (j, _film->log(), false));
 485                 } catch (...) {
 486                         return a;
 487                 }
 488
 489                 int64_t offset = 0;
 490                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 491
 492                         DCPOMATIC_ASSERT (j->video_frame_rate ());
 493                         double const cfr = j->video_frame_rate().get();
 494                         Frame const trim_start = j->trim_start().frames_round (cfr);
 495                         Frame const trim_end = j->trim_end().frames_round (cfr);
 496                         int const ffr = _film->video_frame_rate ();
 497
 498                         DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
 499                         if (j->reference_video ()) {
 500                                 shared_ptr<dcp::ReelAsset> ra = k->main_picture ();
 501                                 DCPOMATIC_ASSERT (ra);
 502                                 ra->set_entry_point (ra->entry_point() + trim_start);
 503                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 504                                 a.push_back (
 505                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 506                                         );
 507                         }
 508
 509                         if (j->reference_audio ()) {
 510                                 shared_ptr<dcp::ReelAsset> ra = k->main_sound ();
 511                                 DCPOMATIC_ASSERT (ra);
 512                                 ra->set_entry_point (ra->entry_point() + trim_start);
 513                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 514                                 a.push_back (
 515                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 516                                         );
 517                         }
 518
 519                         if (j->reference_subtitle ()) {
 520                                 shared_ptr<dcp::ReelAsset> ra = k->main_subtitle ();
 521                                 DCPOMATIC_ASSERT (ra);
 522                                 ra->set_entry_point (ra->entry_point() + trim_start);
 523                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 524                                 a.push_back (
 525                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 526                                         );
 527                         }
 528
 529                         /* Assume that main picture duration is the length of the reel */
 530                         offset += k->main_picture()->duration ();
 531                 }
 532         }
 533
 534         return a;
 535 }
 536
 537 bool
 538 Player::pass ()
 539 {
 540         if (!_have_valid_pieces) {
 541                 setup_pieces ();
 542         }
 543
 544         if (_playlist->length() == DCPTime()) {
 545                 /* Special case of an empty Film; just give one black frame */
 546                 emit_video (black_player_video_frame(EYES_BOTH), DCPTime());
 547                 return true;
 548         }
 549
 550         /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
 551
 552         shared_ptr<Piece> earliest_content;
 553         optional<DCPTime> earliest_time;
 554
 555         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 556                 if (i->done) {
 557                         continue;
 558                 }
 559
 560                 DCPTime const t = content_time_to_dcp (i, i->decoder->position());
 561                 if (t > i->content->end()) {
 562                         i->done = true;
 563                 } else {
 564
 565                         /* Given two choices at the same time, pick the one with a subtitle so we see it before
 566                            the video.
 567                         */
 568                         if (!earliest_time || t < *earliest_time || (t == *earliest_time && i->decoder->subtitle)) {
 569                                 earliest_time = t;
 570                                 earliest_content = i;
 571                         }
 572                 }
 573         }
 574
 575         bool done = false;
 576
 577         enum {
 578                 NONE,
 579                 CONTENT,
 580                 BLACK,
 581                 SILENT
 582         } which = NONE;
 583
 584         if (earliest_content) {
 585                 which = CONTENT;
 586         }
 587
 588         if (!_black.done() && (!earliest_time || _black.position() < *earliest_time)) {
 589                 earliest_time = _black.position ();
 590                 which = BLACK;
 591         }
 592
 593         if (!_silent.done() && (!earliest_time || _silent.position() < *earliest_time)) {
 594                 earliest_time = _silent.position ();
 595                 which = SILENT;
 596         }
 597
 598         switch (which) {
 599         case CONTENT:
 600                 earliest_content->done = earliest_content->decoder->pass ();
 601                 break;
 602         case BLACK:
 603                 emit_video (black_player_video_frame(EYES_BOTH), _black.position());
 604                 _black.set_position (_black.position() + one_video_frame());
 605                 break;
 606         case SILENT:
 607         {
 608                 DCPTimePeriod period (_silent.period_at_position());
 609                 if (_last_audio_time) {
 610                         /* Sometimes the thing that happened last finishes fractionally before
 611                            this silence.  Bodge the start time of the silence to fix it.  I'm
 612                            not sure if this is the right solution --- maybe the last thing should
 613                            be padded `forward' rather than this thing padding `back'.
 614                         */
 615                         period.from = min(period.from, *_last_audio_time);
 616                 }
 617                 if (period.duration() > one_video_frame()) {
 618                         period.to = period.from + one_video_frame();
 619                 }
 620                 fill_audio (period);
 621                 _silent.set_position (period.to);
 622                 break;
 623         }
 624         case NONE:
 625                 done = true;
 626                 break;
 627         }
 628
 629         /* Emit any audio that is ready */
 630
 631         /* Work out the time before which the audio is definitely all here.  This is the earliest last_push_end of one
 632            of our streams, or the position of the _silent.
 633         */
 634         DCPTime pull_to = _film->length ();
 635         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 636                 if (!i->second.piece->done && i->second.last_push_end < pull_to) {
 637                         pull_to = i->second.last_push_end;
 638                 }
 639         }
 640         if (!_silent.done() && _silent.position() < pull_to) {
 641                 pull_to = _silent.position();
 642         }
 643
 644         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 645         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 646                 if (_last_audio_time && i->second < *_last_audio_time) {
 647                         /* This new data comes before the last we emitted (or the last seek); discard it */
 648                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 649                         if (!cut.first) {
 650                                 continue;
 651                         }
 652                         *i = cut;
 653                 } else if (_last_audio_time && i->second > *_last_audio_time) {
 654                         /* There's a gap between this data and the last we emitted; fill with silence */
 655                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 656                 }
 657
 658                 emit_audio (i->first, i->second);
 659         }
 660
 661         if (done) {
 662                 _shuffler->flush ();
 663                 for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 664                         do_emit_video(i->first, i->second);
 665                 }
 666         }
 667
 668         return done;
 669 }
 670
 671 optional<PositionImage>
 672 Player::subtitles_for_frame (DCPTime time) const
 673 {
 674         list<PositionImage> subtitles;
 675
 676         int const vfr = _film->video_frame_rate();
 677
 678         BOOST_FOREACH (PlayerSubtitles i, _active_subtitles.get_burnt (DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), _always_burn_subtitles)) {
 679
 680                 /* Image subtitles */
 681                 list<PositionImage> c = transform_image_subtitles (i.image);
 682                 copy (c.begin(), c.end(), back_inserter (subtitles));
 683
 684                 /* Text subtitles (rendered to an image) */
 685                 if (!i.text.empty ()) {
 686                         list<PositionImage> s = render_subtitles (i.text, i.fonts, _video_container_size, time, vfr);
 687                         copy (s.begin(), s.end(), back_inserter (subtitles));
 688                 }
 689         }
 690
 691         if (subtitles.empty ()) {
 692                 return optional<PositionImage> ();
 693         }
 694
 695         return merge (subtitles);
 696 }
 697
 698 void
 699 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 700 {
 701         shared_ptr<Piece> piece = wp.lock ();
 702         if (!piece) {
 703                 return;
 704         }
 705
 706         FrameRateChange frc(piece->content->active_video_frame_rate(), _film->video_frame_rate());
 707         if (frc.skip && (video.frame % 2) == 1) {
 708                 return;
 709         }
 710
 711         /* Time of the first frame we will emit */
 712         DCPTime const time = content_video_to_dcp (piece, video.frame);
 713
 714         /* Discard if it's before the content's period or the last accurate seek.  We can't discard
 715            if it's after the content's period here as in that case we still need to fill any gap between
 716            `now' and the end of the content's period.
 717         */
 718         if (time < piece->content->position() || (_last_video_time && time < *_last_video_time)) {
 719                 return;
 720         }
 721
 722         /* Fill gaps that we discover now that we have some video which needs to be emitted.
 723            This is where we need to fill to.
 724         */
 725         DCPTime fill_to = min (time, piece->content->end());
 726
 727         if (_last_video_time) {
 728                 DCPTime fill_from = max (*_last_video_time, piece->content->position());
 729                 LastVideoMap::const_iterator last = _last_video.find (wp);
 730                 if (_film->three_d()) {
 731                         DCPTime j = fill_from;
 732                         Eyes eyes = _last_video_eyes.get_value_or(EYES_LEFT);
 733                         if (eyes == EYES_BOTH) {
 734                                 eyes = EYES_LEFT;
 735                         }
 736                         while (j < fill_to || eyes != video.eyes) {
 737                                 if (last != _last_video.end()) {
 738                                         shared_ptr<PlayerVideo> copy = last->second->shallow_copy();
 739                                         copy->set_eyes (eyes);
 740                                         emit_video (copy, j);
 741                                 } else {
 742                                         emit_video (black_player_video_frame(eyes), j);
 743                                 }
 744                                 if (eyes == EYES_RIGHT) {
 745                                         j += one_video_frame();
 746                                 }
 747                                 eyes = increment_eyes (eyes);
 748                         }
 749                 } else {
 750                         for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
 751                                 if (last != _last_video.end()) {
 752                                         emit_video (last->second, j);
 753                                 } else {
 754                                         emit_video (black_player_video_frame(EYES_BOTH), j);
 755                                 }
 756                         }
 757                 }
 758         }
 759
 760         _last_video[wp].reset (
 761                 new PlayerVideo (
 762                         video.image,
 763                         piece->content->video->crop (),
 764                         piece->content->video->fade (video.frame),
 765                         piece->content->video->scale().size (
 766                                 piece->content->video, _video_container_size, _film->frame_size ()
 767                                 ),
 768                         _video_container_size,
 769                         video.eyes,
 770                         video.part,
 771                         piece->content->video->colour_conversion(),
 772                         piece->content,
 773                         video.frame
 774                         )
 775                 );
 776
 777         DCPTime t = time;
 778         for (int i = 0; i < frc.repeat; ++i) {
 779                 if (t < piece->content->end()) {
 780                         emit_video (_last_video[wp], t);
 781                 }
 782                 t += one_video_frame ();
 783         }
 784 }
 785
 786 void
 787 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 788 {
 789         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 790
 791         shared_ptr<Piece> piece = wp.lock ();
 792         if (!piece) {
 793                 return;
 794         }
 795
 796         shared_ptr<AudioContent> content = piece->content->audio;
 797         DCPOMATIC_ASSERT (content);
 798
 799         /* Compute time in the DCP */
 800         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame);
 801         /* And the end of this block in the DCP */
 802         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), content->resampled_frame_rate());
 803
 804         /* Remove anything that comes before the start or after the end of the content */
 805         if (time < piece->content->position()) {
 806                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 807                 if (!cut.first) {
 808                         /* This audio is entirely discarded */
 809                         return;
 810                 }
 811                 content_audio.audio = cut.first;
 812                 time = cut.second;
 813         } else if (time > piece->content->end()) {
 814                 /* Discard it all */
 815                 return;
 816         } else if (end > piece->content->end()) {
 817                 Frame const remaining_frames = DCPTime(piece->content->end() - time).frames_round(_film->audio_frame_rate());
 818                 if (remaining_frames == 0) {
 819                         return;
 820                 }
 821                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 822                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 823                 content_audio.audio = cut;
 824         }
 825
 826         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 827
 828         /* Gain */
 829
 830         if (content->gain() != 0) {
 831                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 832                 gain->apply_gain (content->gain ());
 833                 content_audio.audio = gain;
 834         }
 835
 836         /* Remap */
 837
 838         content_audio.audio = remap (content_audio.audio, _film->audio_channels(), stream->mapping());
 839
 840         /* Process */
 841
 842         if (_audio_processor) {
 843                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 844         }
 845
 846         /* Push */
 847
 848         _audio_merger.push (content_audio.audio, time);
 849         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 850         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 851 }
 852
 853 void
 854 Player::image_subtitle_start (weak_ptr<Piece> wp, ContentImageSubtitle subtitle)
 855 {
 856         shared_ptr<Piece> piece = wp.lock ();
 857         if (!piece) {
 858                 return;
 859         }
 860
 861         /* Apply content's subtitle offsets */
 862         subtitle.sub.rectangle.x += piece->content->subtitle->x_offset ();
 863         subtitle.sub.rectangle.y += piece->content->subtitle->y_offset ();
 864
 865         /* Apply content's subtitle scale */
 866         subtitle.sub.rectangle.width *= piece->content->subtitle->x_scale ();
 867         subtitle.sub.rectangle.height *= piece->content->subtitle->y_scale ();
 868
 869         /* Apply a corrective translation to keep the subtitle centred after that scale */
 870         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * (piece->content->subtitle->x_scale() - 1);
 871         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * (piece->content->subtitle->y_scale() - 1);
 872
 873         PlayerSubtitles ps;
 874         ps.image.push_back (subtitle.sub);
 875         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 876
 877         _active_subtitles.add_from (wp, ps, from);
 878 }
 879
 880 void
 881 Player::text_subtitle_start (weak_ptr<Piece> wp, ContentTextSubtitle subtitle)
 882 {
 883         shared_ptr<Piece> piece = wp.lock ();
 884         if (!piece) {
 885                 return;
 886         }
 887
 888         PlayerSubtitles ps;
 889         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 890
 891         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 892                 s.set_h_position (s.h_position() + piece->content->subtitle->x_offset ());
 893                 s.set_v_position (s.v_position() + piece->content->subtitle->y_offset ());
 894                 float const xs = piece->content->subtitle->x_scale();
 895                 float const ys = piece->content->subtitle->y_scale();
 896                 float size = s.size();
 897
 898                 /* Adjust size to express the common part of the scaling;
 899                    e.g. if xs = ys = 0.5 we scale size by 2.
 900                 */
 901                 if (xs > 1e-5 && ys > 1e-5) {
 902                         size *= 1 / min (1 / xs, 1 / ys);
 903                 }
 904                 s.set_size (size);
 905
 906                 /* Then express aspect ratio changes */
 907                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 908                         s.set_aspect_adjust (xs / ys);
 909                 }
 910
 911                 s.set_in (dcp::Time(from.seconds(), 1000));
 912                 ps.text.push_back (SubtitleString (s, piece->content->subtitle->outline_width()));
 913                 ps.add_fonts (piece->content->subtitle->fonts ());
 914         }
 915
 916         _active_subtitles.add_from (wp, ps, from);
 917 }
 918
 919 void
 920 Player::subtitle_stop (weak_ptr<Piece> wp, ContentTime to)
 921 {
 922         if (!_active_subtitles.have (wp)) {
 923                 return;
 924         }
 925
 926         shared_ptr<Piece> piece = wp.lock ();
 927         if (!piece) {
 928                 return;
 929         }
 930
 931         DCPTime const dcp_to = content_time_to_dcp (piece, to);
 932
 933         pair<PlayerSubtitles, DCPTime> from = _active_subtitles.add_to (wp, dcp_to);
 934
 935         if (piece->content->subtitle->use() && !_always_burn_subtitles && !piece->content->subtitle->burn()) {
 936                 Subtitle (from.first, DCPTimePeriod (from.second, dcp_to));
 937         }
 938 }
 939
 940 void
 941 Player::seek (DCPTime time, bool accurate)
 942 {
 943         if (!_have_valid_pieces) {
 944                 setup_pieces ();
 945         }
 946
 947         if (_shuffler) {
 948                 _shuffler->clear ();
 949         }
 950
 951         _delay.clear ();
 952
 953         if (_audio_processor) {
 954                 _audio_processor->flush ();
 955         }
 956
 957         _audio_merger.clear ();
 958         _active_subtitles.clear ();
 959
 960         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 961                 if (time < i->content->position()) {
 962                         /* Before; seek to 0 */
 963                         i->decoder->seek (ContentTime(), accurate);
 964                         i->done = false;
 965                 } else if (i->content->position() <= time && time < i->content->end()) {
 966                         /* During; seek to position */
 967                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
 968                         i->done = false;
 969                 } else {
 970                         /* After; this piece is done */
 971                         i->done = true;
 972                 }
 973         }
 974
 975         if (accurate) {
 976                 _last_video_time = time;
 977                 _last_video_eyes = EYES_LEFT;
 978                 _last_audio_time = time;
 979         } else {
 980                 _last_video_time = optional<DCPTime>();
 981                 _last_video_eyes = optional<Eyes>();
 982                 _last_audio_time = optional<DCPTime>();
 983         }
 984
 985         _black.set_position (time);
 986         _silent.set_position (time);
 987
 988         _last_video.clear ();
 989 }
 990
 991 void
 992 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
 993 {
 994         /* We need a delay to give a little wiggle room to ensure that relevent subtitles arrive at the
 995            player before the video that requires them.
 996         */
 997         _delay.push_back (make_pair (pv, time));
 998
 999         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1000                 _last_video_time = time + one_video_frame();
1001         }
1002         _last_video_eyes = increment_eyes (pv->eyes());
1003
1004         if (_delay.size() < 3) {
1005                 return;
1006         }
1007
1008         pair<shared_ptr<PlayerVideo>, DCPTime> to_do = _delay.front();
1009         _delay.pop_front();
1010         do_emit_video (to_do.first, to_do.second);
1011 }
1012
1013 void
1014 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1015 {
1016         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1017                 _active_subtitles.clear_before (time);
1018         }
1019
1020         optional<PositionImage> subtitles = subtitles_for_frame (time);
1021         if (subtitles) {
1022                 pv->set_subtitle (subtitles.get ());
1023         }
1024
1025         Video (pv, time);
1026 }
1027
1028 void
1029 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1030 {
1031         /* Log if the assert below is about to fail */
1032         if (_last_audio_time && time != *_last_audio_time) {
1033                 _film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_last_audio_time)), LogEntry::TYPE_WARNING);
1034         }
1035
1036         /* This audio must follow on from the previous */
1037         DCPOMATIC_ASSERT (!_last_audio_time || time == *_last_audio_time);
1038         Audio (data, time);
1039         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate());
1040 }
1041
1042 void
1043 Player::fill_audio (DCPTimePeriod period)
1044 {
1045         if (period.from == period.to) {
1046                 return;
1047         }
1048
1049         DCPOMATIC_ASSERT (period.from < period.to);
1050
1051         DCPTime t = period.from;
1052         while (t < period.to) {
1053                 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1054                 Frame const samples = block.frames_round(_film->audio_frame_rate());
1055                 if (samples) {
1056                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1057                         silence->make_silent ();
1058                         emit_audio (silence, t);
1059                 }
1060                 t += block;
1061         }
1062 }
1063
1064 DCPTime
1065 Player::one_video_frame () const
1066 {
1067         return DCPTime::from_frames (1, _film->video_frame_rate ());
1068 }
1069
1070 pair<shared_ptr<AudioBuffers>, DCPTime>
1071 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1072 {
1073         DCPTime const discard_time = discard_to - time;
1074         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1075         Frame remaining_frames = audio->frames() - discard_frames;
1076         if (remaining_frames <= 0) {
1077                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1078         }
1079         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1080         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1081         return make_pair(cut, time + discard_time);
1082 }
1083
1084 void
1085 Player::set_dcp_decode_reduction (optional<int> reduction)
1086 {
1087         if (reduction == _dcp_decode_reduction) {
1088                 return;
1089         }
1090
1091         _dcp_decode_reduction = reduction;
1092         _have_valid_pieces = false;
1093         Changed (PlayerProperty::DCP_DECODE_REDUCTION, false);
1094 }