src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2018 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_text.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "text_content.h"
  44 #include "text_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "dcp_decoder.h"
  48 #include "image_decoder.h"
  49 #include "compose.hpp"
  50 #include "shuffler.h"
  51 #include <dcp/reel.h>
  52 #include <dcp/reel_sound_asset.h>
  53 #include <dcp/reel_subtitle_asset.h>
  54 #include <dcp/reel_picture_asset.h>
  55 #include <dcp/reel_closed_caption_asset.h>
  56 #include <boost/foreach.hpp>
  57 #include <stdint.h>
  58 #include <algorithm>
  59 #include <iostream>
  60
  61 #include "i18n.h"
  62
  63 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
  64
  65 using std::list;
  66 using std::cout;
  67 using std::min;
  68 using std::max;
  69 using std::min;
  70 using std::vector;
  71 using std::pair;
  72 using std::map;
  73 using std::make_pair;
  74 using std::copy;
  75 using boost::shared_ptr;
  76 using boost::weak_ptr;
  77 using boost::dynamic_pointer_cast;
  78 using boost::optional;
  79 using boost::scoped_ptr;
  80
  81 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
  82 int const PlayerProperty::PLAYLIST = 701;
  83 int const PlayerProperty::FILM_CONTAINER = 702;
  84 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
  85 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
  86
  87 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  88         : _film (film)
  89         , _playlist (playlist)
  90         , _can_run (false)
  91         , _ignore_video (false)
  92         , _ignore_audio (false)
  93         , _ignore_text (false)
  94         , _always_burn_open_subtitles (false)
  95         , _fast (false)
  96         , _play_referenced (false)
  97         , _audio_merger (_film->audio_frame_rate())
  98         , _shuffler (0)
  99 {
 100         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
 101         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 102         _playlist_content_may_change_connection = _playlist->ContentMayChange.connect (bind(&Player::playlist_content_may_change, this));
 103         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind(&Player::playlist_content_changed, this, _1, _2, _3));
 104         _playlist_content_not_changed_connection = _playlist->ContentNotChanged.connect (bind(&Player::playlist_content_not_changed, this));
 105         set_video_container_size (_film->frame_size ());
 106
 107         film_changed (Film::AUDIO_PROCESSOR);
 108
 109         setup_pieces ();
 110         seek (DCPTime (), true);
 111 }
 112
 113 Player::~Player ()
 114 {
 115         delete _shuffler;
 116 }
 117
 118 void
 119 Player::setup_pieces ()
 120 {
 121         boost::mutex::scoped_lock lm (_mutex);
 122         setup_pieces_unlocked ();
 123 }
 124
 125 void
 126 Player::setup_pieces_unlocked ()
 127 {
 128         _pieces.clear ();
 129
 130         delete _shuffler;
 131         _shuffler = new Shuffler();
 132         _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
 133
 134         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 135
 136                 if (!i->paths_valid ()) {
 137                         continue;
 138                 }
 139
 140                 if (_ignore_video && _ignore_audio && i->text.empty()) {
 141                         /* We're only interested in text and this content has none */
 142                         continue;
 143                 }
 144
 145                 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log(), _fast);
 146                 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
 147
 148                 if (!decoder) {
 149                         /* Not something that we can decode; e.g. Atmos content */
 150                         continue;
 151                 }
 152
 153                 if (decoder->video && _ignore_video) {
 154                         decoder->video->set_ignore (true);
 155                 }
 156
 157                 if (decoder->audio && _ignore_audio) {
 158                         decoder->audio->set_ignore (true);
 159                 }
 160
 161                 if (_ignore_text) {
 162                         BOOST_FOREACH (shared_ptr<TextDecoder> i, decoder->text) {
 163                                 i->set_ignore (true);
 164                         }
 165                 }
 166
 167                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 168                 if (dcp) {
 169                         dcp->set_decode_referenced (_play_referenced);
 170                         if (_play_referenced) {
 171                                 dcp->set_forced_reduction (_dcp_decode_reduction);
 172                         }
 173                 }
 174
 175                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 176                 _pieces.push_back (piece);
 177
 178                 if (decoder->video) {
 179                         if (i->video->frame_type() == VIDEO_FRAME_TYPE_3D_LEFT || i->video->frame_type() == VIDEO_FRAME_TYPE_3D_RIGHT) {
 180                                 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
 181                                 decoder->video->Data.connect (bind (&Shuffler::video, _shuffler, weak_ptr<Piece>(piece), _1));
 182                         } else {
 183                                 decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece>(piece), _1));
 184                         }
 185                 }
 186
 187                 if (decoder->audio) {
 188                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 189                 }
 190
 191                 list<shared_ptr<TextDecoder> >::const_iterator j = decoder->text.begin();
 192
 193                 while (j != decoder->text.end()) {
 194                         (*j)->BitmapStart.connect (
 195                                 bind(&Player::bitmap_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 196                                 );
 197                         (*j)->PlainStart.connect (
 198                                 bind(&Player::plain_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
 199                                 );
 200                         (*j)->Stop.connect (
 201                                 bind(&Player::subtitle_stop, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1, _2)
 202                                 );
 203
 204                         ++j;
 205                 }
 206         }
 207
 208         _stream_states.clear ();
 209         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 210                 if (i->content->audio) {
 211                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 212                                 _stream_states[j] = StreamState (i, i->content->position ());
 213                         }
 214                 }
 215         }
 216
 217         _black = Empty (_film->content(), _film->length(), bind(&Content::video, _1));
 218         _silent = Empty (_film->content(), _film->length(), bind(&Content::audio, _1));
 219
 220         _last_video_time = DCPTime ();
 221         _last_video_eyes = EYES_BOTH;
 222         _last_audio_time = DCPTime ();
 223         _can_run = true;
 224 }
 225
 226 void
 227 Player::playlist_content_may_change ()
 228 {
 229         {
 230                 boost::mutex::scoped_lock lm (_mutex);
 231                 /* The player content is probably about to change, so we can't carry on
 232                    until that has happened and we've rebuilt our pieces.  Stop pass()
 233                    and seek() from working until then.
 234                 */
 235                 _can_run = false;
 236         }
 237
 238         MayChange ();
 239 }
 240
 241 void
 242 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
 243 {
 244         /* A change in our content has gone through.  Re-build our pieces and signal
 245            it to anybody that is interested.
 246         */
 247
 248         shared_ptr<Content> c = w.lock ();
 249         if (!c) {
 250                 return;
 251         }
 252
 253         setup_pieces ();
 254
 255         if (
 256                 property == ContentProperty::POSITION ||
 257                 property == ContentProperty::LENGTH ||
 258                 property == ContentProperty::TRIM_START ||
 259                 property == ContentProperty::TRIM_END ||
 260                 property == ContentProperty::PATH ||
 261                 property == VideoContentProperty::FRAME_TYPE ||
 262                 property == VideoContentProperty::COLOUR_CONVERSION ||
 263                 property == AudioContentProperty::STREAMS ||
 264                 property == DCPContentProperty::NEEDS_ASSETS ||
 265                 property == DCPContentProperty::NEEDS_KDM ||
 266                 property == DCPContentProperty::CPL ||
 267                 property == TextContentProperty::COLOUR ||
 268                 property == TextContentProperty::EFFECT ||
 269                 property == TextContentProperty::EFFECT_COLOUR ||
 270                 property == FFmpegContentProperty::SUBTITLE_STREAM ||
 271                 property == FFmpegContentProperty::FILTERS ||
 272                 property == TextContentProperty::LINE_SPACING ||
 273                 property == TextContentProperty::OUTLINE_WIDTH ||
 274                 property == TextContentProperty::Y_SCALE ||
 275                 property == TextContentProperty::FADE_IN ||
 276                 property == TextContentProperty::FADE_OUT ||
 277                 property == ContentProperty::VIDEO_FRAME_RATE ||
 278                 property == TextContentProperty::USE ||
 279                 property == TextContentProperty::X_OFFSET ||
 280                 property == TextContentProperty::Y_OFFSET ||
 281                 property == TextContentProperty::X_SCALE ||
 282                 property == TextContentProperty::FONTS ||
 283                 property == TextContentProperty::TYPE ||
 284                 property == VideoContentProperty::CROP ||
 285                 property == VideoContentProperty::SCALE ||
 286                 property == VideoContentProperty::FADE_IN ||
 287                 property == VideoContentProperty::FADE_OUT
 288                 ) {
 289
 290                 Changed (property, frequent);
 291         }
 292 }
 293
 294 void
 295 Player::playlist_content_not_changed ()
 296 {
 297         /* A possible content change did end up happening for some reason */
 298         setup_pieces ();
 299         NotChanged ();
 300 }
 301
 302 void
 303 Player::set_video_container_size (dcp::Size s)
 304 {
 305         {
 306                 boost::mutex::scoped_lock lm (_mutex);
 307
 308                 if (s == _video_container_size) {
 309                         return;
 310                 }
 311
 312                 _video_container_size = s;
 313
 314                 _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 315                 _black_image->make_black ();
 316         }
 317
 318         Changed (PlayerProperty::VIDEO_CONTAINER_SIZE, false);
 319 }
 320
 321 void
 322 Player::playlist_changed ()
 323 {
 324         setup_pieces ();
 325         Changed (PlayerProperty::PLAYLIST, false);
 326 }
 327
 328 void
 329 Player::film_changed (Film::Property p)
 330 {
 331         /* Here we should notice Film properties that affect our output, and
 332            alert listeners that our output now would be different to how it was
 333            last time we were run.
 334         */
 335
 336         if (p == Film::CONTAINER) {
 337                 Changed (PlayerProperty::FILM_CONTAINER, false);
 338         } else if (p == Film::VIDEO_FRAME_RATE) {
 339                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 340                    so we need new pieces here.
 341                 */
 342                 setup_pieces ();
 343                 Changed (PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
 344         } else if (p == Film::AUDIO_PROCESSOR) {
 345                 if (_film->audio_processor ()) {
 346                         boost::mutex::scoped_lock lm (_mutex);
 347                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 348                 }
 349         } else if (p == Film::AUDIO_CHANNELS) {
 350                 boost::mutex::scoped_lock lm (_mutex);
 351                 _audio_merger.clear ();
 352         }
 353 }
 354
 355 list<PositionImage>
 356 Player::transform_bitmap_texts (list<BitmapText> subs) const
 357 {
 358         list<PositionImage> all;
 359
 360         for (list<BitmapText>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 361                 if (!i->image) {
 362                         continue;
 363                 }
 364
 365                 /* We will scale the subtitle up to fit _video_container_size */
 366                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
 367
 368                 all.push_back (
 369                         PositionImage (
 370                                 i->image->scale (
 371                                         scaled_size,
 372                                         dcp::YUV_TO_RGB_REC601,
 373                                         i->image->pixel_format (),
 374                                         true,
 375                                         _fast
 376                                         ),
 377                                 Position<int> (
 378                                         lrint (_video_container_size.width * i->rectangle.x),
 379                                         lrint (_video_container_size.height * i->rectangle.y)
 380                                         )
 381                                 )
 382                         );
 383         }
 384
 385         return all;
 386 }
 387
 388 shared_ptr<PlayerVideo>
 389 Player::black_player_video_frame (Eyes eyes) const
 390 {
 391         return shared_ptr<PlayerVideo> (
 392                 new PlayerVideo (
 393                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 394                         Crop (),
 395                         optional<double> (),
 396                         _video_container_size,
 397                         _video_container_size,
 398                         eyes,
 399                         PART_WHOLE,
 400                         PresetColourConversion::all().front().conversion,
 401                         boost::weak_ptr<Content>(),
 402                         boost::optional<Frame>()
 403                 )
 404         );
 405 }
 406
 407 Frame
 408 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 409 {
 410         DCPTime s = t - piece->content->position ();
 411         s = min (piece->content->length_after_trim(), s);
 412         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 413
 414         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 415            then convert that ContentTime to frames at the content's rate.  However this fails for
 416            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 417            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 418
 419            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 420         */
 421         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 422 }
 423
 424 DCPTime
 425 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 426 {
 427         /* See comment in dcp_to_content_video */
 428         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime(piece->content->trim_start(), piece->frc);
 429         return d + piece->content->position();
 430 }
 431
 432 Frame
 433 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 434 {
 435         DCPTime s = t - piece->content->position ();
 436         s = min (piece->content->length_after_trim(), s);
 437         /* See notes in dcp_to_content_video */
 438         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 439 }
 440
 441 DCPTime
 442 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 443 {
 444         /* See comment in dcp_to_content_video */
 445         return DCPTime::from_frames (f, _film->audio_frame_rate())
 446                 - DCPTime (piece->content->trim_start(), piece->frc)
 447                 + piece->content->position();
 448 }
 449
 450 ContentTime
 451 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 452 {
 453         DCPTime s = t - piece->content->position ();
 454         s = min (piece->content->length_after_trim(), s);
 455         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 456 }
 457
 458 DCPTime
 459 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 460 {
 461         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 462 }
 463
 464 list<shared_ptr<Font> >
 465 Player::get_subtitle_fonts ()
 466 {
 467         boost::mutex::scoped_lock lm (_mutex);
 468
 469         list<shared_ptr<Font> > fonts;
 470         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 471                 BOOST_FOREACH (shared_ptr<TextContent> j, i->content->text) {
 472                         /* XXX: things may go wrong if there are duplicate font IDs
 473                            with different font files.
 474                         */
 475                         list<shared_ptr<Font> > f = j->fonts ();
 476                         copy (f.begin(), f.end(), back_inserter (fonts));
 477                 }
 478         }
 479
 480         return fonts;
 481 }
 482
 483 /** Set this player never to produce any video data */
 484 void
 485 Player::set_ignore_video ()
 486 {
 487         boost::mutex::scoped_lock lm (_mutex);
 488         _ignore_video = true;
 489         setup_pieces_unlocked ();
 490 }
 491
 492 void
 493 Player::set_ignore_audio ()
 494 {
 495         boost::mutex::scoped_lock lm (_mutex);
 496         _ignore_audio = true;
 497         setup_pieces_unlocked ();
 498 }
 499
 500 void
 501 Player::set_ignore_text ()
 502 {
 503         boost::mutex::scoped_lock lm (_mutex);
 504         _ignore_text = true;
 505         setup_pieces_unlocked ();
 506 }
 507
 508 /** Set the player to always burn open texts into the image regardless of the content settings */
 509 void
 510 Player::set_always_burn_open_subtitles ()
 511 {
 512         boost::mutex::scoped_lock lm (_mutex);
 513         _always_burn_open_subtitles = true;
 514 }
 515
 516 /** Sets up the player to be faster, possibly at the expense of quality */
 517 void
 518 Player::set_fast ()
 519 {
 520         boost::mutex::scoped_lock lm (_mutex);
 521         _fast = true;
 522         setup_pieces_unlocked ();
 523 }
 524
 525 void
 526 Player::set_play_referenced ()
 527 {
 528         boost::mutex::scoped_lock lm (_mutex);
 529         _play_referenced = true;
 530         setup_pieces_unlocked ();
 531 }
 532
 533 list<ReferencedReelAsset>
 534 Player::get_reel_assets ()
 535 {
 536         /* Does not require a lock on _mutex as it's only called from DCPEncoder */
 537
 538         list<ReferencedReelAsset> a;
 539
 540         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 541                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 542                 if (!j) {
 543                         continue;
 544                 }
 545
 546                 scoped_ptr<DCPDecoder> decoder;
 547                 try {
 548                         decoder.reset (new DCPDecoder (j, _film->log(), false));
 549                 } catch (...) {
 550                         return a;
 551                 }
 552
 553                 int64_t offset = 0;
 554                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 555
 556                         DCPOMATIC_ASSERT (j->video_frame_rate ());
 557                         double const cfr = j->video_frame_rate().get();
 558                         Frame const trim_start = j->trim_start().frames_round (cfr);
 559                         Frame const trim_end = j->trim_end().frames_round (cfr);
 560                         int const ffr = _film->video_frame_rate ();
 561
 562                         DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
 563                         if (j->reference_video ()) {
 564                                 shared_ptr<dcp::ReelAsset> ra = k->main_picture ();
 565                                 DCPOMATIC_ASSERT (ra);
 566                                 ra->set_entry_point (ra->entry_point() + trim_start);
 567                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 568                                 a.push_back (
 569                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 570                                         );
 571                         }
 572
 573                         if (j->reference_audio ()) {
 574                                 shared_ptr<dcp::ReelAsset> ra = k->main_sound ();
 575                                 DCPOMATIC_ASSERT (ra);
 576                                 ra->set_entry_point (ra->entry_point() + trim_start);
 577                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 578                                 a.push_back (
 579                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 580                                         );
 581                         }
 582
 583                         if (j->reference_text (TEXT_OPEN_SUBTITLE)) {
 584                                 shared_ptr<dcp::ReelAsset> ra = k->main_subtitle ();
 585                                 DCPOMATIC_ASSERT (ra);
 586                                 ra->set_entry_point (ra->entry_point() + trim_start);
 587                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 588                                 a.push_back (
 589                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 590                                         );
 591                         }
 592
 593                         if (j->reference_text (TEXT_CLOSED_CAPTION)) {
 594                                 shared_ptr<dcp::ReelAsset> ra = k->closed_caption ();
 595                                 DCPOMATIC_ASSERT (ra);
 596                                 ra->set_entry_point (ra->entry_point() + trim_start);
 597                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 598                                 a.push_back (
 599                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 600                                         );
 601                         }
 602
 603                         /* Assume that main picture duration is the length of the reel */
 604                         offset += k->main_picture()->duration ();
 605                 }
 606         }
 607
 608         return a;
 609 }
 610
 611 bool
 612 Player::pass ()
 613 {
 614         boost::mutex::scoped_lock lm (_mutex);
 615
 616         if (!_can_run) {
 617                 /* We can't pass in this state */
 618                 return false;
 619         }
 620
 621         if (_playlist->length() == DCPTime()) {
 622                 /* Special case of an empty Film; just give one black frame */
 623                 emit_video (black_player_video_frame(EYES_BOTH), DCPTime());
 624                 return true;
 625         }
 626
 627         /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
 628
 629         shared_ptr<Piece> earliest_content;
 630         optional<DCPTime> earliest_time;
 631
 632         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 633                 if (i->done) {
 634                         continue;
 635                 }
 636
 637                 DCPTime const t = content_time_to_dcp (i, max(i->decoder->position(), i->content->trim_start()));
 638                 if (t > i->content->end()) {
 639                         i->done = true;
 640                 } else {
 641
 642                         /* Given two choices at the same time, pick the one with texts so we see it before
 643                            the video.
 644                         */
 645                         if (!earliest_time || t < *earliest_time || (t == *earliest_time && !i->decoder->text.empty())) {
 646                                 earliest_time = t;
 647                                 earliest_content = i;
 648                         }
 649                 }
 650         }
 651
 652         bool done = false;
 653
 654         enum {
 655                 NONE,
 656                 CONTENT,
 657                 BLACK,
 658                 SILENT
 659         } which = NONE;
 660
 661         if (earliest_content) {
 662                 which = CONTENT;
 663         }
 664
 665         if (!_black.done() && (!earliest_time || _black.position() < *earliest_time)) {
 666                 earliest_time = _black.position ();
 667                 which = BLACK;
 668         }
 669
 670         if (!_silent.done() && (!earliest_time || _silent.position() < *earliest_time)) {
 671                 earliest_time = _silent.position ();
 672                 which = SILENT;
 673         }
 674
 675         switch (which) {
 676         case CONTENT:
 677                 earliest_content->done = earliest_content->decoder->pass ();
 678                 break;
 679         case BLACK:
 680                 emit_video (black_player_video_frame(EYES_BOTH), _black.position());
 681                 _black.set_position (_black.position() + one_video_frame());
 682                 break;
 683         case SILENT:
 684         {
 685                 DCPTimePeriod period (_silent.period_at_position());
 686                 if (_last_audio_time) {
 687                         /* Sometimes the thing that happened last finishes fractionally before
 688                            this silence.  Bodge the start time of the silence to fix it.  I'm
 689                            not sure if this is the right solution --- maybe the last thing should
 690                            be padded `forward' rather than this thing padding `back'.
 691                         */
 692                         period.from = min(period.from, *_last_audio_time);
 693                 }
 694                 if (period.duration() > one_video_frame()) {
 695                         period.to = period.from + one_video_frame();
 696                 }
 697                 fill_audio (period);
 698                 _silent.set_position (period.to);
 699                 break;
 700         }
 701         case NONE:
 702                 done = true;
 703                 break;
 704         }
 705
 706         /* Emit any audio that is ready */
 707
 708         /* Work out the time before which the audio is definitely all here.  This is the earliest last_push_end of one
 709            of our streams, or the position of the _silent.
 710         */
 711         DCPTime pull_to = _film->length ();
 712         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 713                 if (!i->second.piece->done && i->second.last_push_end < pull_to) {
 714                         pull_to = i->second.last_push_end;
 715                 }
 716         }
 717         if (!_silent.done() && _silent.position() < pull_to) {
 718                 pull_to = _silent.position();
 719         }
 720
 721         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 722         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 723                 if (_last_audio_time && i->second < *_last_audio_time) {
 724                         /* This new data comes before the last we emitted (or the last seek); discard it */
 725                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 726                         if (!cut.first) {
 727                                 continue;
 728                         }
 729                         *i = cut;
 730                 } else if (_last_audio_time && i->second > *_last_audio_time) {
 731                         /* There's a gap between this data and the last we emitted; fill with silence */
 732                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 733                 }
 734
 735                 emit_audio (i->first, i->second);
 736         }
 737
 738         if (done) {
 739                 _shuffler->flush ();
 740                 for (list<pair<shared_ptr<PlayerVideo>, DCPTime> >::const_iterator i = _delay.begin(); i != _delay.end(); ++i) {
 741                         do_emit_video(i->first, i->second);
 742                 }
 743         }
 744
 745         return done;
 746 }
 747
 748 /** @return Open subtitles for the frame at the given time, converted to images */
 749 optional<PositionImage>
 750 Player::open_subtitles_for_frame (DCPTime time) const
 751 {
 752         list<PositionImage> captions;
 753         int const vfr = _film->video_frame_rate();
 754
 755         BOOST_FOREACH (
 756                 PlayerText j,
 757                 _active_texts[TEXT_OPEN_SUBTITLE].get_burnt(DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), _always_burn_open_subtitles)
 758                 ) {
 759
 760                 /* Bitmap subtitles */
 761                 list<PositionImage> c = transform_bitmap_texts (j.bitmap);
 762                 copy (c.begin(), c.end(), back_inserter (captions));
 763
 764                 /* String subtitles (rendered to an image) */
 765                 if (!j.string.empty ()) {
 766                         list<PositionImage> s = render_text (j.string, j.fonts, _video_container_size, time, vfr);
 767                         copy (s.begin(), s.end(), back_inserter (captions));
 768                 }
 769         }
 770
 771         if (captions.empty ()) {
 772                 return optional<PositionImage> ();
 773         }
 774
 775         return merge (captions);
 776 }
 777
 778 void
 779 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 780 {
 781         shared_ptr<Piece> piece = wp.lock ();
 782         if (!piece) {
 783                 return;
 784         }
 785
 786         FrameRateChange frc(piece->content->active_video_frame_rate(), _film->video_frame_rate());
 787         if (frc.skip && (video.frame % 2) == 1) {
 788                 return;
 789         }
 790
 791         /* Time of the first frame we will emit */
 792         DCPTime const time = content_video_to_dcp (piece, video.frame);
 793
 794         /* Discard if it's before the content's period or the last accurate seek.  We can't discard
 795            if it's after the content's period here as in that case we still need to fill any gap between
 796            `now' and the end of the content's period.
 797         */
 798         if (time < piece->content->position() || (_last_video_time && time < *_last_video_time)) {
 799                 return;
 800         }
 801
 802         /* Fill gaps that we discover now that we have some video which needs to be emitted.
 803            This is where we need to fill to.
 804         */
 805         DCPTime fill_to = min (time, piece->content->end());
 806
 807         if (_last_video_time) {
 808                 DCPTime fill_from = max (*_last_video_time, piece->content->position());
 809                 LastVideoMap::const_iterator last = _last_video.find (wp);
 810                 if (_film->three_d()) {
 811                         Eyes fill_to_eyes = video.eyes;
 812                         if (fill_to == piece->content->end()) {
 813                                 /* Don't fill after the end of the content */
 814                                 fill_to_eyes = EYES_LEFT;
 815                         }
 816                         DCPTime j = fill_from;
 817                         Eyes eyes = _last_video_eyes.get_value_or(EYES_LEFT);
 818                         if (eyes == EYES_BOTH) {
 819                                 eyes = EYES_LEFT;
 820                         }
 821                         while (j < fill_to || eyes != fill_to_eyes) {
 822                                 if (last != _last_video.end()) {
 823                                         shared_ptr<PlayerVideo> copy = last->second->shallow_copy();
 824                                         copy->set_eyes (eyes);
 825                                         emit_video (copy, j);
 826                                 } else {
 827                                         emit_video (black_player_video_frame(eyes), j);
 828                                 }
 829                                 if (eyes == EYES_RIGHT) {
 830                                         j += one_video_frame();
 831                                 }
 832                                 eyes = increment_eyes (eyes);
 833                         }
 834                 } else {
 835                         for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
 836                                 if (last != _last_video.end()) {
 837                                         emit_video (last->second, j);
 838                                 } else {
 839                                         emit_video (black_player_video_frame(EYES_BOTH), j);
 840                                 }
 841                         }
 842                 }
 843         }
 844
 845         _last_video[wp].reset (
 846                 new PlayerVideo (
 847                         video.image,
 848                         piece->content->video->crop (),
 849                         piece->content->video->fade (video.frame),
 850                         piece->content->video->scale().size (
 851                                 piece->content->video, _video_container_size, _film->frame_size ()
 852                                 ),
 853                         _video_container_size,
 854                         video.eyes,
 855                         video.part,
 856                         piece->content->video->colour_conversion(),
 857                         piece->content,
 858                         video.frame
 859                         )
 860                 );
 861
 862         DCPTime t = time;
 863         for (int i = 0; i < frc.repeat; ++i) {
 864                 if (t < piece->content->end()) {
 865                         emit_video (_last_video[wp], t);
 866                 }
 867                 t += one_video_frame ();
 868         }
 869 }
 870
 871 void
 872 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 873 {
 874         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 875
 876         shared_ptr<Piece> piece = wp.lock ();
 877         if (!piece) {
 878                 return;
 879         }
 880
 881         shared_ptr<AudioContent> content = piece->content->audio;
 882         DCPOMATIC_ASSERT (content);
 883
 884         /* Compute time in the DCP */
 885         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame);
 886         /* And the end of this block in the DCP */
 887         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), content->resampled_frame_rate());
 888
 889         /* Remove anything that comes before the start or after the end of the content */
 890         if (time < piece->content->position()) {
 891                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 892                 if (!cut.first) {
 893                         /* This audio is entirely discarded */
 894                         return;
 895                 }
 896                 content_audio.audio = cut.first;
 897                 time = cut.second;
 898         } else if (time > piece->content->end()) {
 899                 /* Discard it all */
 900                 return;
 901         } else if (end > piece->content->end()) {
 902                 Frame const remaining_frames = DCPTime(piece->content->end() - time).frames_round(_film->audio_frame_rate());
 903                 if (remaining_frames == 0) {
 904                         return;
 905                 }
 906                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 907                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 908                 content_audio.audio = cut;
 909         }
 910
 911         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 912
 913         /* Gain */
 914
 915         if (content->gain() != 0) {
 916                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 917                 gain->apply_gain (content->gain ());
 918                 content_audio.audio = gain;
 919         }
 920
 921         /* Remap */
 922
 923         content_audio.audio = remap (content_audio.audio, _film->audio_channels(), stream->mapping());
 924
 925         /* Process */
 926
 927         if (_audio_processor) {
 928                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 929         }
 930
 931         /* Push */
 932
 933         _audio_merger.push (content_audio.audio, time);
 934         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 935         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 936 }
 937
 938 void
 939 Player::bitmap_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentBitmapText subtitle)
 940 {
 941         shared_ptr<Piece> piece = wp.lock ();
 942         shared_ptr<const TextContent> text = wc.lock ();
 943         if (!piece || !text) {
 944                 return;
 945         }
 946
 947         /* Apply content's subtitle offsets */
 948         subtitle.sub.rectangle.x += text->x_offset ();
 949         subtitle.sub.rectangle.y += text->y_offset ();
 950
 951         /* Apply a corrective translation to keep the subtitle centred after the scale that is coming up */
 952         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * ((text->x_scale() - 1) / 2);
 953         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * ((text->y_scale() - 1) / 2);
 954
 955         /* Apply content's subtitle scale */
 956         subtitle.sub.rectangle.width *= text->x_scale ();
 957         subtitle.sub.rectangle.height *= text->y_scale ();
 958
 959         PlayerText ps;
 960         ps.bitmap.push_back (subtitle.sub);
 961         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 962
 963         _active_texts[subtitle.type()].add_from (wc, ps, from);
 964 }
 965
 966 void
 967 Player::plain_text_start (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentStringText subtitle)
 968 {
 969         shared_ptr<Piece> piece = wp.lock ();
 970         shared_ptr<const TextContent> text = wc.lock ();
 971         if (!piece || !text) {
 972                 return;
 973         }
 974
 975         PlayerText ps;
 976         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 977
 978         if (from > piece->content->end()) {
 979                 return;
 980         }
 981
 982         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 983                 s.set_h_position (s.h_position() + text->x_offset ());
 984                 s.set_v_position (s.v_position() + text->y_offset ());
 985                 float const xs = text->x_scale();
 986                 float const ys = text->y_scale();
 987                 float size = s.size();
 988
 989                 /* Adjust size to express the common part of the scaling;
 990                    e.g. if xs = ys = 0.5 we scale size by 2.
 991                 */
 992                 if (xs > 1e-5 && ys > 1e-5) {
 993                         size *= 1 / min (1 / xs, 1 / ys);
 994                 }
 995                 s.set_size (size);
 996
 997                 /* Then express aspect ratio changes */
 998                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 999                         s.set_aspect_adjust (xs / ys);
1000                 }
1001
1002                 s.set_in (dcp::Time(from.seconds(), 1000));
1003                 ps.string.push_back (StringText (s, text->outline_width()));
1004                 ps.add_fonts (text->fonts ());
1005         }
1006
1007         _active_texts[subtitle.type()].add_from (wc, ps, from);
1008 }
1009
1010 void
1011 Player::subtitle_stop (weak_ptr<Piece> wp, weak_ptr<const TextContent> wc, ContentTime to, TextType type)
1012 {
1013         if (!_active_texts[type].have (wc)) {
1014                 return;
1015         }
1016
1017         shared_ptr<Piece> piece = wp.lock ();
1018         shared_ptr<const TextContent> text = wc.lock ();
1019         if (!piece || !text) {
1020                 return;
1021         }
1022
1023         DCPTime const dcp_to = content_time_to_dcp (piece, to);
1024
1025         if (dcp_to > piece->content->end()) {
1026                 return;
1027         }
1028
1029         pair<PlayerText, DCPTime> from = _active_texts[type].add_to (wc, dcp_to);
1030
1031         bool const always = (type == TEXT_OPEN_SUBTITLE && _always_burn_open_subtitles);
1032         if (text->use() && !always && !text->burn()) {
1033                 Text (from.first, type, DCPTimePeriod (from.second, dcp_to));
1034         }
1035 }
1036
1037 void
1038 Player::seek (DCPTime time, bool accurate)
1039 {
1040         boost::mutex::scoped_lock lm (_mutex);
1041
1042         if (!_can_run) {
1043                 /* We can't seek in this state */
1044                 return;
1045         }
1046
1047         if (_shuffler) {
1048                 _shuffler->clear ();
1049         }
1050
1051         _delay.clear ();
1052
1053         if (_audio_processor) {
1054                 _audio_processor->flush ();
1055         }
1056
1057         _audio_merger.clear ();
1058         for (int i = 0; i < TEXT_COUNT; ++i) {
1059                 _active_texts[i].clear ();
1060         }
1061
1062         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1063                 if (time < i->content->position()) {
1064                         /* Before; seek to the start of the content */
1065                         i->decoder->seek (dcp_to_content_time (i, i->content->position()), accurate);
1066                         i->done = false;
1067                 } else if (i->content->position() <= time && time < i->content->end()) {
1068                         /* During; seek to position */
1069                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
1070                         i->done = false;
1071                 } else {
1072                         /* After; this piece is done */
1073                         i->done = true;
1074                 }
1075         }
1076
1077         if (accurate) {
1078                 _last_video_time = time;
1079                 _last_video_eyes = EYES_LEFT;
1080                 _last_audio_time = time;
1081         } else {
1082                 _last_video_time = optional<DCPTime>();
1083                 _last_video_eyes = optional<Eyes>();
1084                 _last_audio_time = optional<DCPTime>();
1085         }
1086
1087         _black.set_position (time);
1088         _silent.set_position (time);
1089
1090         _last_video.clear ();
1091 }
1092
1093 void
1094 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1095 {
1096         /* We need a delay to give a little wiggle room to ensure that relevent subtitles arrive at the
1097            player before the video that requires them.
1098         */
1099         _delay.push_back (make_pair (pv, time));
1100
1101         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1102                 _last_video_time = time + one_video_frame();
1103         }
1104         _last_video_eyes = increment_eyes (pv->eyes());
1105
1106         if (_delay.size() < 3) {
1107                 return;
1108         }
1109
1110         pair<shared_ptr<PlayerVideo>, DCPTime> to_do = _delay.front();
1111         _delay.pop_front();
1112         do_emit_video (to_do.first, to_do.second);
1113 }
1114
1115 void
1116 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1117 {
1118         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1119                 for (int i = 0; i < TEXT_COUNT; ++i) {
1120                         _active_texts[i].clear_before (time);
1121                 }
1122         }
1123
1124         optional<PositionImage> subtitles = open_subtitles_for_frame (time);
1125         if (subtitles) {
1126                 pv->set_text (subtitles.get ());
1127         }
1128
1129         Video (pv, time);
1130 }
1131
1132 void
1133 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1134 {
1135         /* Log if the assert below is about to fail */
1136         if (_last_audio_time && time != *_last_audio_time) {
1137                 _film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_last_audio_time)), LogEntry::TYPE_WARNING);
1138         }
1139
1140         /* This audio must follow on from the previous */
1141         DCPOMATIC_ASSERT (!_last_audio_time || time == *_last_audio_time);
1142         Audio (data, time);
1143         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate());
1144 }
1145
1146 void
1147 Player::fill_audio (DCPTimePeriod period)
1148 {
1149         if (period.from == period.to) {
1150                 return;
1151         }
1152
1153         DCPOMATIC_ASSERT (period.from < period.to);
1154
1155         DCPTime t = period.from;
1156         while (t < period.to) {
1157                 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1158                 Frame const samples = block.frames_round(_film->audio_frame_rate());
1159                 if (samples) {
1160                         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1161                         silence->make_silent ();
1162                         emit_audio (silence, t);
1163                 }
1164                 t += block;
1165         }
1166 }
1167
1168 DCPTime
1169 Player::one_video_frame () const
1170 {
1171         return DCPTime::from_frames (1, _film->video_frame_rate ());
1172 }
1173
1174 pair<shared_ptr<AudioBuffers>, DCPTime>
1175 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1176 {
1177         DCPTime const discard_time = discard_to - time;
1178         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1179         Frame remaining_frames = audio->frames() - discard_frames;
1180         if (remaining_frames <= 0) {
1181                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1182         }
1183         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1184         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1185         return make_pair(cut, time + discard_time);
1186 }
1187
1188 void
1189 Player::set_dcp_decode_reduction (optional<int> reduction)
1190 {
1191         {
1192                 boost::mutex::scoped_lock lm (_mutex);
1193
1194                 if (reduction == _dcp_decode_reduction) {
1195                         return;
1196                 }
1197
1198                 _dcp_decode_reduction = reduction;
1199                 setup_pieces_unlocked ();
1200         }
1201
1202         Changed (PlayerProperty::DCP_DECODE_REDUCTION, false);
1203 }
1204
1205 optional<DCPTime>
1206 Player::content_time_to_dcp (shared_ptr<Content> content, ContentTime t)
1207 {
1208         boost::mutex::scoped_lock lm (_mutex);
1209
1210         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
1211                 if (i->content == content) {
1212                         return content_time_to_dcp (i, t);
1213                 }
1214         }
1215
1216         /* We couldn't find this content; perhaps things are being changed over */
1217         return optional<DCPTime>();
1218 }