src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2017 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_subtitles.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "subtitle_content.h"
  44 #include "subtitle_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "content_subtitle.h"
  48 #include "dcp_decoder.h"
  49 #include "image_decoder.h"
  50 #include "resampler.h"
  51 #include "compose.hpp"
  52 #include <dcp/reel.h>
  53 #include <dcp/reel_sound_asset.h>
  54 #include <dcp/reel_subtitle_asset.h>
  55 #include <dcp/reel_picture_asset.h>
  56 #include <boost/foreach.hpp>
  57 #include <stdint.h>
  58 #include <algorithm>
  59 #include <iostream>
  60
  61 #include "i18n.h"
  62
  63 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
  64
  65 using std::list;
  66 using std::cout;
  67 using std::min;
  68 using std::max;
  69 using std::min;
  70 using std::vector;
  71 using std::pair;
  72 using std::map;
  73 using std::make_pair;
  74 using std::copy;
  75 using boost::shared_ptr;
  76 using boost::weak_ptr;
  77 using boost::dynamic_pointer_cast;
  78 using boost::optional;
  79 using boost::scoped_ptr;
  80
  81 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  82         : _film (film)
  83         , _playlist (playlist)
  84         , _have_valid_pieces (false)
  85         , _ignore_video (false)
  86         , _ignore_audio (false)
  87         , _always_burn_subtitles (false)
  88         , _fast (false)
  89         , _play_referenced (false)
  90         , _audio_merger (_film->audio_frame_rate())
  91 {
  92         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  93         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  94         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
  95         set_video_container_size (_film->frame_size ());
  96
  97         film_changed (Film::AUDIO_PROCESSOR);
  98
  99         seek (DCPTime (), true);
 100 }
 101
 102 void
 103 Player::setup_pieces ()
 104 {
 105         _pieces.clear ();
 106
 107         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 108
 109                 if (!i->paths_valid ()) {
 110                         continue;
 111                 }
 112
 113                 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log());
 114                 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
 115
 116                 if (!decoder) {
 117                         /* Not something that we can decode; e.g. Atmos content */
 118                         continue;
 119                 }
 120
 121                 if (decoder->video && _ignore_video) {
 122                         decoder->video->set_ignore ();
 123                 }
 124
 125                 if (decoder->audio && _ignore_audio) {
 126                         decoder->audio->set_ignore ();
 127                 }
 128
 129                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 130                 if (dcp && _play_referenced) {
 131                         dcp->set_decode_referenced ();
 132                 }
 133
 134                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 135                 _pieces.push_back (piece);
 136
 137                 if (decoder->video) {
 138                         decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece> (piece), _1));
 139                 }
 140
 141                 if (decoder->audio) {
 142                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 143                 }
 144
 145                 if (decoder->subtitle) {
 146                         decoder->subtitle->ImageStart.connect (bind (&Player::image_subtitle_start, this, weak_ptr<Piece> (piece), _1));
 147                         decoder->subtitle->TextStart.connect (bind (&Player::text_subtitle_start, this, weak_ptr<Piece> (piece), _1));
 148                         decoder->subtitle->Stop.connect (bind (&Player::subtitle_stop, this, weak_ptr<Piece> (piece), _1));
 149                 }
 150         }
 151
 152         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 153                 if (i->content->audio) {
 154                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 155                                 _stream_states[j] = StreamState (i, i->content->position ());
 156                         }
 157                 }
 158         }
 159
 160         if (!_play_referenced) {
 161                 BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 162                         shared_ptr<DCPContent> dc = dynamic_pointer_cast<DCPContent> (i->content);
 163                         if (dc) {
 164                                 if (dc->reference_video()) {
 165                                         _no_video.push_back (DCPTimePeriod (dc->position(), dc->end()));
 166                                 }
 167                                 if (dc->reference_audio()) {
 168                                         _no_audio.push_back (DCPTimePeriod (dc->position(), dc->end()));
 169                                 }
 170                         }
 171                 }
 172         }
 173
 174         _last_video_time = optional<DCPTime> ();
 175         _last_audio_time = optional<DCPTime> ();
 176         _have_valid_pieces = true;
 177 }
 178
 179 void
 180 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
 181 {
 182         shared_ptr<Content> c = w.lock ();
 183         if (!c) {
 184                 return;
 185         }
 186
 187         if (
 188                 property == ContentProperty::POSITION ||
 189                 property == ContentProperty::LENGTH ||
 190                 property == ContentProperty::TRIM_START ||
 191                 property == ContentProperty::TRIM_END ||
 192                 property == ContentProperty::PATH ||
 193                 property == VideoContentProperty::FRAME_TYPE ||
 194                 property == DCPContentProperty::NEEDS_ASSETS ||
 195                 property == DCPContentProperty::NEEDS_KDM ||
 196                 property == SubtitleContentProperty::COLOUR ||
 197                 property == SubtitleContentProperty::OUTLINE ||
 198                 property == SubtitleContentProperty::SHADOW ||
 199                 property == SubtitleContentProperty::EFFECT_COLOUR ||
 200                 property == FFmpegContentProperty::SUBTITLE_STREAM ||
 201                 property == VideoContentProperty::COLOUR_CONVERSION
 202                 ) {
 203
 204                 _have_valid_pieces = false;
 205                 Changed (frequent);
 206
 207         } else if (
 208                 property == SubtitleContentProperty::LINE_SPACING ||
 209                 property == SubtitleContentProperty::OUTLINE_WIDTH ||
 210                 property == SubtitleContentProperty::Y_SCALE ||
 211                 property == SubtitleContentProperty::FADE_IN ||
 212                 property == SubtitleContentProperty::FADE_OUT ||
 213                 property == ContentProperty::VIDEO_FRAME_RATE ||
 214                 property == SubtitleContentProperty::USE ||
 215                 property == SubtitleContentProperty::X_OFFSET ||
 216                 property == SubtitleContentProperty::Y_OFFSET ||
 217                 property == SubtitleContentProperty::X_SCALE ||
 218                 property == SubtitleContentProperty::FONTS ||
 219                 property == VideoContentProperty::CROP ||
 220                 property == VideoContentProperty::SCALE ||
 221                 property == VideoContentProperty::FADE_IN ||
 222                 property == VideoContentProperty::FADE_OUT
 223                 ) {
 224
 225                 Changed (frequent);
 226         }
 227 }
 228
 229 void
 230 Player::set_video_container_size (dcp::Size s)
 231 {
 232         if (s == _video_container_size) {
 233                 return;
 234         }
 235
 236         _video_container_size = s;
 237
 238         _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 239         _black_image->make_black ();
 240
 241         Changed (false);
 242 }
 243
 244 void
 245 Player::playlist_changed ()
 246 {
 247         _have_valid_pieces = false;
 248         Changed (false);
 249 }
 250
 251 void
 252 Player::film_changed (Film::Property p)
 253 {
 254         /* Here we should notice Film properties that affect our output, and
 255            alert listeners that our output now would be different to how it was
 256            last time we were run.
 257         */
 258
 259         if (p == Film::CONTAINER) {
 260                 Changed (false);
 261         } else if (p == Film::VIDEO_FRAME_RATE) {
 262                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 263                    so we need new pieces here.
 264                 */
 265                 _have_valid_pieces = false;
 266                 Changed (false);
 267         } else if (p == Film::AUDIO_PROCESSOR) {
 268                 if (_film->audio_processor ()) {
 269                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 270                 }
 271         }
 272 }
 273
 274 list<PositionImage>
 275 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
 276 {
 277         list<PositionImage> all;
 278
 279         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 280                 if (!i->image) {
 281                         continue;
 282                 }
 283
 284                 /* We will scale the subtitle up to fit _video_container_size */
 285                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
 286
 287                 /* Then we need a corrective translation, consisting of two parts:
 288                  *
 289                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 290                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 291                  *
 292                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 293                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
 294                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
 295                  *
 296                  * Combining these two translations gives these expressions.
 297                  */
 298
 299                 all.push_back (
 300                         PositionImage (
 301                                 i->image->scale (
 302                                         scaled_size,
 303                                         dcp::YUV_TO_RGB_REC601,
 304                                         i->image->pixel_format (),
 305                                         true,
 306                                         _fast
 307                                         ),
 308                                 Position<int> (
 309                                         lrint (_video_container_size.width * i->rectangle.x),
 310                                         lrint (_video_container_size.height * i->rectangle.y)
 311                                         )
 312                                 )
 313                         );
 314         }
 315
 316         return all;
 317 }
 318
 319 shared_ptr<PlayerVideo>
 320 Player::black_player_video_frame () const
 321 {
 322         return shared_ptr<PlayerVideo> (
 323                 new PlayerVideo (
 324                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 325                         Crop (),
 326                         optional<double> (),
 327                         _video_container_size,
 328                         _video_container_size,
 329                         EYES_BOTH,
 330                         PART_WHOLE,
 331                         PresetColourConversion::all().front().conversion
 332                 )
 333         );
 334 }
 335
 336 Frame
 337 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 338 {
 339         DCPTime s = t - piece->content->position ();
 340         s = min (piece->content->length_after_trim(), s);
 341         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 342
 343         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 344            then convert that ContentTime to frames at the content's rate.  However this fails for
 345            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 346            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 347
 348            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 349         */
 350         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 351 }
 352
 353 DCPTime
 354 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 355 {
 356         /* See comment in dcp_to_content_video */
 357         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime (piece->content->trim_start (), piece->frc);
 358         return max (DCPTime (), d + piece->content->position ());
 359 }
 360
 361 Frame
 362 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 363 {
 364         DCPTime s = t - piece->content->position ();
 365         s = min (piece->content->length_after_trim(), s);
 366         /* See notes in dcp_to_content_video */
 367         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 368 }
 369
 370 DCPTime
 371 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 372 {
 373         /* See comment in dcp_to_content_video */
 374         DCPTime const d = DCPTime::from_frames (f, _film->audio_frame_rate()) - DCPTime (piece->content->trim_start(), piece->frc);
 375         return max (DCPTime (), d + piece->content->position ());
 376 }
 377
 378 ContentTime
 379 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 380 {
 381         DCPTime s = t - piece->content->position ();
 382         s = min (piece->content->length_after_trim(), s);
 383         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 384 }
 385
 386 DCPTime
 387 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 388 {
 389         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 390 }
 391
 392 list<shared_ptr<Font> >
 393 Player::get_subtitle_fonts ()
 394 {
 395         if (!_have_valid_pieces) {
 396                 setup_pieces ();
 397         }
 398
 399         list<shared_ptr<Font> > fonts;
 400         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
 401                 if (p->content->subtitle) {
 402                         /* XXX: things may go wrong if there are duplicate font IDs
 403                            with different font files.
 404                         */
 405                         list<shared_ptr<Font> > f = p->content->subtitle->fonts ();
 406                         copy (f.begin(), f.end(), back_inserter (fonts));
 407                 }
 408         }
 409
 410         return fonts;
 411 }
 412
 413 /** Set this player never to produce any video data */
 414 void
 415 Player::set_ignore_video ()
 416 {
 417         _ignore_video = true;
 418 }
 419
 420 /** Set whether or not this player should always burn text subtitles into the image,
 421  *  regardless of the content settings.
 422  *  @param burn true to always burn subtitles, false to obey content settings.
 423  */
 424 void
 425 Player::set_always_burn_subtitles (bool burn)
 426 {
 427         _always_burn_subtitles = burn;
 428 }
 429
 430 void
 431 Player::set_fast ()
 432 {
 433         _fast = true;
 434         _have_valid_pieces = false;
 435 }
 436
 437 void
 438 Player::set_play_referenced ()
 439 {
 440         _play_referenced = true;
 441         _have_valid_pieces = false;
 442 }
 443
 444 list<ReferencedReelAsset>
 445 Player::get_reel_assets ()
 446 {
 447         list<ReferencedReelAsset> a;
 448
 449         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 450                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 451                 if (!j) {
 452                         continue;
 453                 }
 454
 455                 scoped_ptr<DCPDecoder> decoder;
 456                 try {
 457                         decoder.reset (new DCPDecoder (j, _film->log()));
 458                 } catch (...) {
 459                         return a;
 460                 }
 461
 462                 int64_t offset = 0;
 463                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 464
 465                         DCPOMATIC_ASSERT (j->video_frame_rate ());
 466                         double const cfr = j->video_frame_rate().get();
 467                         Frame const trim_start = j->trim_start().frames_round (cfr);
 468                         Frame const trim_end = j->trim_end().frames_round (cfr);
 469                         int const ffr = _film->video_frame_rate ();
 470
 471                         DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
 472                         if (j->reference_video ()) {
 473                                 shared_ptr<dcp::ReelAsset> ra = k->main_picture ();
 474                                 DCPOMATIC_ASSERT (ra);
 475                                 ra->set_entry_point (ra->entry_point() + trim_start);
 476                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 477                                 a.push_back (
 478                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 479                                         );
 480                         }
 481
 482                         if (j->reference_audio ()) {
 483                                 shared_ptr<dcp::ReelAsset> ra = k->main_sound ();
 484                                 DCPOMATIC_ASSERT (ra);
 485                                 ra->set_entry_point (ra->entry_point() + trim_start);
 486                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 487                                 a.push_back (
 488                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 489                                         );
 490                         }
 491
 492                         if (j->reference_subtitle ()) {
 493                                 shared_ptr<dcp::ReelAsset> ra = k->main_subtitle ();
 494                                 DCPOMATIC_ASSERT (ra);
 495                                 ra->set_entry_point (ra->entry_point() + trim_start);
 496                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 497                                 a.push_back (
 498                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 499                                         );
 500                         }
 501
 502                         /* Assume that main picture duration is the length of the reel */
 503                         offset += k->main_picture()->duration ();
 504                 }
 505         }
 506
 507         return a;
 508 }
 509
 510 bool
 511 Player::pass ()
 512 {
 513         if (!_have_valid_pieces) {
 514                 setup_pieces ();
 515         }
 516
 517         shared_ptr<Piece> earliest;
 518         DCPTime earliest_content;
 519
 520         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 521                 if (!i->done) {
 522                         DCPTime const t = i->content->position() + DCPTime (i->decoder->position(), i->frc);
 523                         if (!earliest || t < earliest_content) {
 524                                 earliest_content = t;
 525                                 earliest = i;
 526                         }
 527                 }
 528         }
 529
 530         if (earliest) {
 531                 earliest->done = earliest->decoder->pass ();
 532                 if (earliest->done && earliest->content->audio) {
 533                         /* Flush the Player audio system for this piece */
 534                         BOOST_FOREACH (AudioStreamPtr i, earliest->content->audio->streams()) {
 535                                 audio_flush (earliest, i);
 536                         }
 537                 }
 538         }
 539
 540         if (_last_video_time) {
 541                 fill_video (DCPTimePeriod (_last_video_time.get(), earliest ? earliest_content : _playlist->length()));
 542         } else if (_last_seek_time) {
 543                 fill_video (DCPTimePeriod (_last_seek_time.get(), _last_seek_time.get() + one_video_frame ()));
 544         }
 545
 546         /* XXX: fill audio */
 547
 548         if (!earliest) {
 549                 return true;
 550         }
 551
 552         /* Emit any audio that is ready */
 553
 554         DCPTime pull_from = _playlist->length ();
 555         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 556                 if (!i->second.piece->done && i->second.last_push_end < pull_from) {
 557                         pull_from = i->second.last_push_end;
 558                 }
 559         }
 560
 561         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_from);
 562         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 563                 if (_last_audio_time && i->second < _last_audio_time.get()) {
 564                         /* There has been an accurate seek and we have received some audio before the seek time;
 565                            discard it.
 566                         */
 567                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 568                         if (!cut.first) {
 569                                 continue;
 570                         }
 571                         *i = cut;
 572                 }
 573
 574                 if (_last_audio_time) {
 575                         /* XXX: does this remain necessary? */
 576                         fill_audio (DCPTimePeriod (_last_audio_time.get(), i->second));
 577                 }
 578
 579                 Audio (i->first, i->second);
 580                 _last_audio_time = i->second + DCPTime::from_frames(i->first->frames(), _film->audio_frame_rate());
 581         }
 582
 583         return false;
 584 }
 585
 586 optional<PositionImage>
 587 Player::subtitles_for_frame (DCPTime time) const
 588 {
 589         list<PositionImage> subtitles;
 590
 591         BOOST_FOREACH (PlayerSubtitles i, _active_subtitles.get_burnt (time, _always_burn_subtitles)) {
 592
 593                 /* Image subtitles */
 594                 list<PositionImage> c = transform_image_subtitles (i.image);
 595                 copy (c.begin(), c.end(), back_inserter (subtitles));
 596
 597                 /* Text subtitles (rendered to an image) */
 598                 if (!i.text.empty ()) {
 599                         list<PositionImage> s = render_subtitles (i.text, i.fonts, _video_container_size, time);
 600                         copy (s.begin(), s.end(), back_inserter (subtitles));
 601                 }
 602         }
 603
 604         if (subtitles.empty ()) {
 605                 return optional<PositionImage> ();
 606         }
 607
 608         return merge (subtitles);
 609 }
 610
 611 void
 612 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 613 {
 614         shared_ptr<Piece> piece = wp.lock ();
 615         if (!piece) {
 616                 return;
 617         }
 618
 619         FrameRateChange frc(piece->content->active_video_frame_rate(), _film->video_frame_rate());
 620         if (frc.skip && (video.frame % 2) == 1) {
 621                 return;
 622         }
 623
 624         /* Time and period of the frame we will emit */
 625         DCPTime const time = content_video_to_dcp (piece, video.frame);
 626         DCPTimePeriod const period (time, time + one_video_frame());
 627
 628         /* Discard if it's outside the content's period or if it's before the last accurate seek */
 629         if (time < piece->content->position() || time >= piece->content->end() || (_last_video_time && time < _last_video_time)) {
 630                 return;
 631         }
 632
 633         /* Fill gaps caused by (the hopefully rare event of) a decoder not emitting contiguous video.  We have to do this here
 634            as in the problematic case we are about to emit a frame which is not contiguous with the previous.
 635         */
 636
 637         if (_last_video_time) {
 638                 fill_video (DCPTimePeriod (_last_video_time.get(), time));
 639         }
 640
 641         _last_video.reset (
 642                 new PlayerVideo (
 643                         video.image,
 644                         piece->content->video->crop (),
 645                         piece->content->video->fade (video.frame),
 646                         piece->content->video->scale().size (
 647                                 piece->content->video, _video_container_size, _film->frame_size ()
 648                                 ),
 649                         _video_container_size,
 650                         video.eyes,
 651                         video.part,
 652                         piece->content->video->colour_conversion ()
 653                         )
 654                 );
 655
 656         optional<PositionImage> subtitles = subtitles_for_frame (time);
 657         if (subtitles) {
 658                 _last_video->set_subtitle (subtitles.get ());
 659         }
 660
 661         Video (_last_video, time);
 662
 663         _last_video_time = time + one_video_frame ();
 664
 665         _active_subtitles.clear_before (time);
 666 }
 667
 668 void
 669 Player::audio_flush (shared_ptr<Piece> piece, AudioStreamPtr stream)
 670 {
 671         shared_ptr<AudioContent> content = piece->content->audio;
 672         DCPOMATIC_ASSERT (content);
 673
 674         shared_ptr<Resampler> r = resampler (content, stream, false);
 675         if (!r) {
 676                 return;
 677         }
 678
 679         pair<shared_ptr<const AudioBuffers>, Frame> ro = r->flush ();
 680         if (ro.first->frames() == 0) {
 681                 return;
 682         }
 683
 684         ContentAudio content_audio;
 685         content_audio.audio = ro.first;
 686         content_audio.frame = ro.second;
 687
 688         /* Compute time in the DCP */
 689         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame) + DCPTime::from_seconds (content->delay() / 1000.0);
 690
 691         audio_transform (content, stream, content_audio, time);
 692 }
 693
 694 /** Do our common processing on some audio */
 695 void
 696 Player::audio_transform (shared_ptr<AudioContent> content, AudioStreamPtr stream, ContentAudio content_audio, DCPTime time)
 697 {
 698         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 699
 700         /* Gain */
 701
 702         if (content->gain() != 0) {
 703                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 704                 gain->apply_gain (content->gain ());
 705                 content_audio.audio = gain;
 706         }
 707
 708         /* Remap */
 709
 710         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), content_audio.audio->frames()));
 711         dcp_mapped->make_silent ();
 712
 713         AudioMapping map = stream->mapping ();
 714         for (int i = 0; i < map.input_channels(); ++i) {
 715                 for (int j = 0; j < dcp_mapped->channels(); ++j) {
 716                         if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 717                                 dcp_mapped->accumulate_channel (
 718                                         content_audio.audio.get(),
 719                                         i,
 720                                         static_cast<dcp::Channel> (j),
 721                                         map.get (i, static_cast<dcp::Channel> (j))
 722                                         );
 723                         }
 724                 }
 725         }
 726
 727         content_audio.audio = dcp_mapped;
 728
 729         /* Process */
 730
 731         if (_audio_processor) {
 732                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 733         }
 734
 735         /* Push */
 736
 737         _audio_merger.push (content_audio.audio, time);
 738         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 739         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 740 }
 741
 742 void
 743 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 744 {
 745         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 746
 747         shared_ptr<Piece> piece = wp.lock ();
 748         if (!piece) {
 749                 return;
 750         }
 751
 752         shared_ptr<AudioContent> content = piece->content->audio;
 753         DCPOMATIC_ASSERT (content);
 754
 755         /* Resample */
 756         if (stream->frame_rate() != content->resampled_frame_rate()) {
 757                 shared_ptr<Resampler> r = resampler (content, stream, true);
 758                 pair<shared_ptr<const AudioBuffers>, Frame> ro = r->run (content_audio.audio, content_audio.frame);
 759                 if (ro.first->frames() == 0) {
 760                         return;
 761                 }
 762                 content_audio.audio = ro.first;
 763                 content_audio.frame = ro.second;
 764         }
 765
 766         /* Compute time in the DCP */
 767         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame) + DCPTime::from_seconds (content->delay() / 1000.0);
 768         /* And the end of this block in the DCP */
 769         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), content->resampled_frame_rate());
 770
 771         /* Remove anything that comes before the start or after the end of the content */
 772         if (time < piece->content->position()) {
 773                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 774                 if (!cut.first) {
 775                         /* This audio is entirely discarded */
 776                         return;
 777                 }
 778                 content_audio.audio = cut.first;
 779                 time = cut.second;
 780         } else if (time > piece->content->end()) {
 781                 /* Discard it all */
 782                 return;
 783         } else if (end > piece->content->end()) {
 784                 Frame const remaining_frames = DCPTime(piece->content->end() - time).frames_round(_film->audio_frame_rate());
 785                 DCPOMATIC_ASSERT (remaining_frames > 0);
 786                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 787                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 788                 content_audio.audio = cut;
 789         }
 790
 791         audio_transform (content, stream, content_audio, time);
 792 }
 793
 794 void
 795 Player::image_subtitle_start (weak_ptr<Piece> wp, ContentImageSubtitle subtitle)
 796 {
 797         shared_ptr<Piece> piece = wp.lock ();
 798         if (!piece) {
 799                 return;
 800         }
 801
 802         /* Apply content's subtitle offsets */
 803         subtitle.sub.rectangle.x += piece->content->subtitle->x_offset ();
 804         subtitle.sub.rectangle.y += piece->content->subtitle->y_offset ();
 805
 806         /* Apply content's subtitle scale */
 807         subtitle.sub.rectangle.width *= piece->content->subtitle->x_scale ();
 808         subtitle.sub.rectangle.height *= piece->content->subtitle->y_scale ();
 809
 810         /* Apply a corrective translation to keep the subtitle centred after that scale */
 811         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * (piece->content->subtitle->x_scale() - 1);
 812         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * (piece->content->subtitle->y_scale() - 1);
 813
 814         PlayerSubtitles ps;
 815         ps.image.push_back (subtitle.sub);
 816         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 817
 818         _active_subtitles.add_from (wp, ps, from);
 819 }
 820
 821 void
 822 Player::text_subtitle_start (weak_ptr<Piece> wp, ContentTextSubtitle subtitle)
 823 {
 824         shared_ptr<Piece> piece = wp.lock ();
 825         if (!piece) {
 826                 return;
 827         }
 828
 829         PlayerSubtitles ps;
 830         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 831
 832         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 833                 s.set_h_position (s.h_position() + piece->content->subtitle->x_offset ());
 834                 s.set_v_position (s.v_position() + piece->content->subtitle->y_offset ());
 835                 float const xs = piece->content->subtitle->x_scale();
 836                 float const ys = piece->content->subtitle->y_scale();
 837                 float size = s.size();
 838
 839                 /* Adjust size to express the common part of the scaling;
 840                    e.g. if xs = ys = 0.5 we scale size by 2.
 841                 */
 842                 if (xs > 1e-5 && ys > 1e-5) {
 843                         size *= 1 / min (1 / xs, 1 / ys);
 844                 }
 845                 s.set_size (size);
 846
 847                 /* Then express aspect ratio changes */
 848                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 849                         s.set_aspect_adjust (xs / ys);
 850                 }
 851
 852                 s.set_in (dcp::Time(from.seconds(), 1000));
 853                 ps.text.push_back (SubtitleString (s, piece->content->subtitle->outline_width()));
 854                 ps.add_fonts (piece->content->subtitle->fonts ());
 855         }
 856
 857         _active_subtitles.add_from (wp, ps, from);
 858 }
 859
 860 void
 861 Player::subtitle_stop (weak_ptr<Piece> wp, ContentTime to)
 862 {
 863         if (!_active_subtitles.have (wp)) {
 864                 return;
 865         }
 866
 867         shared_ptr<Piece> piece = wp.lock ();
 868         if (!piece) {
 869                 return;
 870         }
 871
 872         DCPTime const dcp_to = content_time_to_dcp (piece, to);
 873
 874         pair<PlayerSubtitles, DCPTime> from = _active_subtitles.add_to (wp, dcp_to);
 875
 876         if (piece->content->subtitle->use() && !_always_burn_subtitles && !piece->content->subtitle->burn()) {
 877                 Subtitle (from.first, DCPTimePeriod (from.second, dcp_to));
 878         }
 879 }
 880
 881 void
 882 Player::seek (DCPTime time, bool accurate)
 883 {
 884         if (_audio_processor) {
 885                 _audio_processor->flush ();
 886         }
 887
 888         for (ResamplerMap::iterator i = _resamplers.begin(); i != _resamplers.end(); ++i) {
 889                 i->second->flush ();
 890                 i->second->reset ();
 891         }
 892
 893         _audio_merger.clear ();
 894         _active_subtitles.clear ();
 895
 896         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 897                 if (time < i->content->position()) {
 898                         /* Before; seek to 0 */
 899                         i->decoder->seek (ContentTime(), accurate);
 900                         i->done = false;
 901                 } else if (i->content->position() <= time && time < i->content->end()) {
 902                         /* During; seek to position */
 903                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
 904                         i->done = false;
 905                 } else {
 906                         /* After; this piece is done */
 907                         i->done = true;
 908                 }
 909         }
 910
 911         if (accurate) {
 912                 _last_video_time = time;
 913                 _last_audio_time = time;
 914         } else {
 915                 _last_video_time = optional<DCPTime> ();
 916                 _last_audio_time = optional<DCPTime> ();
 917         }
 918
 919         _last_seek_time = time;
 920 }
 921
 922 shared_ptr<Resampler>
 923 Player::resampler (shared_ptr<const AudioContent> content, AudioStreamPtr stream, bool create)
 924 {
 925         ResamplerMap::const_iterator i = _resamplers.find (make_pair (content, stream));
 926         if (i != _resamplers.end ()) {
 927                 return i->second;
 928         }
 929
 930         if (!create) {
 931                 return shared_ptr<Resampler> ();
 932         }
 933
 934         LOG_GENERAL (
 935                 "Creating new resampler from %1 to %2 with %3 channels",
 936                 stream->frame_rate(),
 937                 content->resampled_frame_rate(),
 938                 stream->channels()
 939                 );
 940
 941         shared_ptr<Resampler> r (
 942                 new Resampler (stream->frame_rate(), content->resampled_frame_rate(), stream->channels())
 943                 );
 944
 945         _resamplers[make_pair(content, stream)] = r;
 946         return r;
 947 }
 948
 949 void
 950 Player::fill_video (DCPTimePeriod period)
 951 {
 952         /* XXX: this may not work for 3D */
 953         BOOST_FOREACH (DCPTimePeriod i, subtract(period, _no_video)) {
 954                 for (DCPTime j = i.from; j < i.to; j += one_video_frame()) {
 955                         if (_playlist->video_content_at(j) && _last_video) {
 956                                 Video (shared_ptr<PlayerVideo> (new PlayerVideo (*_last_video)), j);
 957                         } else {
 958                                 shared_ptr<PlayerVideo> black = black_player_video_frame ();
 959                                 optional<PositionImage> subtitles = subtitles_for_frame (j);
 960                                 if (subtitles) {
 961                                         black->set_subtitle (subtitles.get ());
 962                                 }
 963                                 Video (black, j);
 964                         }
 965                         _last_video_time = j;
 966                 }
 967         }
 968 }
 969
 970 void
 971 Player::fill_audio (DCPTimePeriod period)
 972 {
 973         BOOST_FOREACH (DCPTimePeriod i, subtract(period, _no_audio)) {
 974                 DCPTime t = i.from;
 975                 while (t < i.to) {
 976                         DCPTime block = min (DCPTime::from_seconds (0.5), i.to - t);
 977                         Frame const samples = block.frames_round(_film->audio_frame_rate());
 978                         if (samples) {
 979                                 shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
 980                                 silence->make_silent ();
 981                                 Audio (silence, t);
 982                         }
 983                         t += block;
 984                 }
 985         }
 986 }
 987
 988 DCPTime
 989 Player::one_video_frame () const
 990 {
 991         return DCPTime::from_frames (1, _film->video_frame_rate ());
 992 }
 993
 994 pair<shared_ptr<AudioBuffers>, DCPTime>
 995 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
 996 {
 997         DCPTime const discard_time = discard_to - time;
 998         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
 999         Frame remaining_frames = audio->frames() - discard_frames;
1000         if (remaining_frames <= 0) {
1001                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1002         }
1003         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1004         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1005         return make_pair(cut, time + discard_time);
1006 }