src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2017 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21 #include "player.h"
  22 #include "film.h"
  23 #include "audio_buffers.h"
  24 #include "content_audio.h"
  25 #include "dcp_content.h"
  26 #include "job.h"
  27 #include "image.h"
  28 #include "raw_image_proxy.h"
  29 #include "ratio.h"
  30 #include "log.h"
  31 #include "render_subtitles.h"
  32 #include "config.h"
  33 #include "content_video.h"
  34 #include "player_video.h"
  35 #include "frame_rate_change.h"
  36 #include "audio_processor.h"
  37 #include "playlist.h"
  38 #include "referenced_reel_asset.h"
  39 #include "decoder_factory.h"
  40 #include "decoder.h"
  41 #include "video_decoder.h"
  42 #include "audio_decoder.h"
  43 #include "subtitle_content.h"
  44 #include "subtitle_decoder.h"
  45 #include "ffmpeg_content.h"
  46 #include "audio_content.h"
  47 #include "content_subtitle.h"
  48 #include "dcp_decoder.h"
  49 #include "image_decoder.h"
  50 #include "resampler.h"
  51 #include "compose.hpp"
  52 #include <dcp/reel.h>
  53 #include <dcp/reel_sound_asset.h>
  54 #include <dcp/reel_subtitle_asset.h>
  55 #include <dcp/reel_picture_asset.h>
  56 #include <boost/foreach.hpp>
  57 #include <stdint.h>
  58 #include <algorithm>
  59 #include <iostream>
  60
  61 #include "i18n.h"
  62
  63 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), LogEntry::TYPE_GENERAL);
  64
  65 using std::list;
  66 using std::cout;
  67 using std::min;
  68 using std::max;
  69 using std::min;
  70 using std::vector;
  71 using std::pair;
  72 using std::map;
  73 using std::make_pair;
  74 using std::copy;
  75 using boost::shared_ptr;
  76 using boost::weak_ptr;
  77 using boost::dynamic_pointer_cast;
  78 using boost::optional;
  79 using boost::scoped_ptr;
  80
  81 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist)
  82         : _film (film)
  83         , _playlist (playlist)
  84         , _have_valid_pieces (false)
  85         , _ignore_video (false)
  86         , _ignore_audio (false)
  87         , _always_burn_subtitles (false)
  88         , _fast (false)
  89         , _play_referenced (false)
  90         , _audio_merger (_film->audio_frame_rate())
  91 {
  92         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  93         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  94         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::playlist_content_changed, this, _1, _2, _3));
  95         set_video_container_size (_film->frame_size ());
  96
  97         film_changed (Film::AUDIO_PROCESSOR);
  98
  99         seek (DCPTime (), true);
 100 }
 101
 102 void
 103 Player::setup_pieces ()
 104 {
 105         _pieces.clear ();
 106
 107         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 108
 109                 if (!i->paths_valid ()) {
 110                         continue;
 111                 }
 112
 113                 shared_ptr<Decoder> decoder = decoder_factory (i, _film->log());
 114                 FrameRateChange frc (i->active_video_frame_rate(), _film->video_frame_rate());
 115
 116                 if (!decoder) {
 117                         /* Not something that we can decode; e.g. Atmos content */
 118                         continue;
 119                 }
 120
 121                 if (decoder->video && _ignore_video) {
 122                         decoder->video->set_ignore ();
 123                 }
 124
 125                 if (decoder->audio && _ignore_audio) {
 126                         decoder->audio->set_ignore ();
 127                 }
 128
 129                 shared_ptr<DCPDecoder> dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
 130                 if (dcp && _play_referenced) {
 131                         dcp->set_decode_referenced ();
 132                 }
 133
 134                 shared_ptr<Piece> piece (new Piece (i, decoder, frc));
 135                 _pieces.push_back (piece);
 136
 137                 if (decoder->video) {
 138                         decoder->video->Data.connect (bind (&Player::video, this, weak_ptr<Piece> (piece), _1));
 139                 }
 140
 141                 if (decoder->audio) {
 142                         decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
 143                 }
 144
 145                 if (decoder->subtitle) {
 146                         decoder->subtitle->ImageStart.connect (bind (&Player::image_subtitle_start, this, weak_ptr<Piece> (piece), _1));
 147                         decoder->subtitle->TextStart.connect (bind (&Player::text_subtitle_start, this, weak_ptr<Piece> (piece), _1));
 148                         decoder->subtitle->Stop.connect (bind (&Player::subtitle_stop, this, weak_ptr<Piece> (piece), _1));
 149                 }
 150         }
 151
 152         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 153                 if (i->content->audio) {
 154                         BOOST_FOREACH (AudioStreamPtr j, i->content->audio->streams()) {
 155                                 _stream_states[j] = StreamState (i, i->content->position ());
 156                         }
 157                 }
 158         }
 159
 160         if (!_play_referenced) {
 161                 BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 162                         shared_ptr<DCPContent> dc = dynamic_pointer_cast<DCPContent> (i->content);
 163                         if (dc) {
 164                                 if (dc->reference_video()) {
 165                                         _no_video.push_back (DCPTimePeriod (dc->position(), dc->end()));
 166                                 }
 167                                 if (dc->reference_audio()) {
 168                                         _no_audio.push_back (DCPTimePeriod (dc->position(), dc->end()));
 169                                 }
 170                         }
 171                 }
 172         }
 173
 174         _last_video_time = DCPTime ();
 175         _last_audio_time = DCPTime ();
 176         _have_valid_pieces = true;
 177 }
 178
 179 void
 180 Player::playlist_content_changed (weak_ptr<Content> w, int property, bool frequent)
 181 {
 182         shared_ptr<Content> c = w.lock ();
 183         if (!c) {
 184                 return;
 185         }
 186
 187         if (
 188                 property == ContentProperty::POSITION ||
 189                 property == ContentProperty::LENGTH ||
 190                 property == ContentProperty::TRIM_START ||
 191                 property == ContentProperty::TRIM_END ||
 192                 property == ContentProperty::PATH ||
 193                 property == VideoContentProperty::FRAME_TYPE ||
 194                 property == DCPContentProperty::NEEDS_ASSETS ||
 195                 property == DCPContentProperty::NEEDS_KDM ||
 196                 property == SubtitleContentProperty::COLOUR ||
 197                 property == SubtitleContentProperty::OUTLINE ||
 198                 property == SubtitleContentProperty::SHADOW ||
 199                 property == SubtitleContentProperty::EFFECT_COLOUR ||
 200                 property == FFmpegContentProperty::SUBTITLE_STREAM ||
 201                 property == VideoContentProperty::COLOUR_CONVERSION
 202                 ) {
 203
 204                 _have_valid_pieces = false;
 205                 Changed (frequent);
 206
 207         } else if (
 208                 property == SubtitleContentProperty::LINE_SPACING ||
 209                 property == SubtitleContentProperty::OUTLINE_WIDTH ||
 210                 property == SubtitleContentProperty::Y_SCALE ||
 211                 property == SubtitleContentProperty::FADE_IN ||
 212                 property == SubtitleContentProperty::FADE_OUT ||
 213                 property == ContentProperty::VIDEO_FRAME_RATE ||
 214                 property == SubtitleContentProperty::USE ||
 215                 property == SubtitleContentProperty::X_OFFSET ||
 216                 property == SubtitleContentProperty::Y_OFFSET ||
 217                 property == SubtitleContentProperty::X_SCALE ||
 218                 property == SubtitleContentProperty::FONTS ||
 219                 property == VideoContentProperty::CROP ||
 220                 property == VideoContentProperty::SCALE ||
 221                 property == VideoContentProperty::FADE_IN ||
 222                 property == VideoContentProperty::FADE_OUT
 223                 ) {
 224
 225                 Changed (frequent);
 226         }
 227 }
 228
 229 void
 230 Player::set_video_container_size (dcp::Size s)
 231 {
 232         if (s == _video_container_size) {
 233                 return;
 234         }
 235
 236         _video_container_size = s;
 237
 238         _black_image.reset (new Image (AV_PIX_FMT_RGB24, _video_container_size, true));
 239         _black_image->make_black ();
 240
 241         Changed (false);
 242 }
 243
 244 void
 245 Player::playlist_changed ()
 246 {
 247         _have_valid_pieces = false;
 248         Changed (false);
 249 }
 250
 251 void
 252 Player::film_changed (Film::Property p)
 253 {
 254         /* Here we should notice Film properties that affect our output, and
 255            alert listeners that our output now would be different to how it was
 256            last time we were run.
 257         */
 258
 259         if (p == Film::CONTAINER) {
 260                 Changed (false);
 261         } else if (p == Film::VIDEO_FRAME_RATE) {
 262                 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
 263                    so we need new pieces here.
 264                 */
 265                 _have_valid_pieces = false;
 266                 Changed (false);
 267         } else if (p == Film::AUDIO_PROCESSOR) {
 268                 if (_film->audio_processor ()) {
 269                         _audio_processor = _film->audio_processor()->clone (_film->audio_frame_rate ());
 270                 }
 271         }
 272 }
 273
 274 list<PositionImage>
 275 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
 276 {
 277         list<PositionImage> all;
 278
 279         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 280                 if (!i->image) {
 281                         continue;
 282                 }
 283
 284                 /* We will scale the subtitle up to fit _video_container_size */
 285                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
 286
 287                 /* Then we need a corrective translation, consisting of two parts:
 288                  *
 289                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 290                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 291                  *
 292                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 293                  *     (width_before_subtitle_scale * (1 - subtitle_x_scale) / 2) and
 294                  *     (height_before_subtitle_scale * (1 - subtitle_y_scale) / 2).
 295                  *
 296                  * Combining these two translations gives these expressions.
 297                  */
 298
 299                 all.push_back (
 300                         PositionImage (
 301                                 i->image->scale (
 302                                         scaled_size,
 303                                         dcp::YUV_TO_RGB_REC601,
 304                                         i->image->pixel_format (),
 305                                         true,
 306                                         _fast
 307                                         ),
 308                                 Position<int> (
 309                                         lrint (_video_container_size.width * i->rectangle.x),
 310                                         lrint (_video_container_size.height * i->rectangle.y)
 311                                         )
 312                                 )
 313                         );
 314         }
 315
 316         return all;
 317 }
 318
 319 shared_ptr<PlayerVideo>
 320 Player::black_player_video_frame () const
 321 {
 322         return shared_ptr<PlayerVideo> (
 323                 new PlayerVideo (
 324                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 325                         Crop (),
 326                         optional<double> (),
 327                         _video_container_size,
 328                         _video_container_size,
 329                         EYES_BOTH,
 330                         PART_WHOLE,
 331                         PresetColourConversion::all().front().conversion
 332                 )
 333         );
 334 }
 335
 336 Frame
 337 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 338 {
 339         DCPTime s = t - piece->content->position ();
 340         s = min (piece->content->length_after_trim(), s);
 341         s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
 342
 343         /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
 344            then convert that ContentTime to frames at the content's rate.  However this fails for
 345            situations like content at 29.9978733fps, DCP at 30fps.  The accuracy of the Time type is not
 346            enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
 347
 348            Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
 349         */
 350         return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
 351 }
 352
 353 DCPTime
 354 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 355 {
 356         /* See comment in dcp_to_content_video */
 357         DCPTime const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime (piece->content->trim_start (), piece->frc);
 358         return max (DCPTime (), d + piece->content->position ());
 359 }
 360
 361 Frame
 362 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
 363 {
 364         DCPTime s = t - piece->content->position ();
 365         s = min (piece->content->length_after_trim(), s);
 366         /* See notes in dcp_to_content_video */
 367         return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 368 }
 369
 370 DCPTime
 371 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
 372 {
 373         /* See comment in dcp_to_content_video */
 374         DCPTime const d = DCPTime::from_frames (f, _film->audio_frame_rate()) - DCPTime (piece->content->trim_start(), piece->frc);
 375         return max (DCPTime (), d + piece->content->position ());
 376 }
 377
 378 ContentTime
 379 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
 380 {
 381         DCPTime s = t - piece->content->position ();
 382         s = min (piece->content->length_after_trim(), s);
 383         return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
 384 }
 385
 386 DCPTime
 387 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
 388 {
 389         return max (DCPTime (), DCPTime (t - piece->content->trim_start(), piece->frc) + piece->content->position());
 390 }
 391
 392 list<shared_ptr<Font> >
 393 Player::get_subtitle_fonts ()
 394 {
 395         if (!_have_valid_pieces) {
 396                 setup_pieces ();
 397         }
 398
 399         list<shared_ptr<Font> > fonts;
 400         BOOST_FOREACH (shared_ptr<Piece>& p, _pieces) {
 401                 if (p->content->subtitle) {
 402                         /* XXX: things may go wrong if there are duplicate font IDs
 403                            with different font files.
 404                         */
 405                         list<shared_ptr<Font> > f = p->content->subtitle->fonts ();
 406                         copy (f.begin(), f.end(), back_inserter (fonts));
 407                 }
 408         }
 409
 410         return fonts;
 411 }
 412
 413 /** Set this player never to produce any video data */
 414 void
 415 Player::set_ignore_video ()
 416 {
 417         _ignore_video = true;
 418 }
 419
 420 /** Set whether or not this player should always burn text subtitles into the image,
 421  *  regardless of the content settings.
 422  *  @param burn true to always burn subtitles, false to obey content settings.
 423  */
 424 void
 425 Player::set_always_burn_subtitles (bool burn)
 426 {
 427         _always_burn_subtitles = burn;
 428 }
 429
 430 void
 431 Player::set_fast ()
 432 {
 433         _fast = true;
 434         _have_valid_pieces = false;
 435 }
 436
 437 void
 438 Player::set_play_referenced ()
 439 {
 440         _play_referenced = true;
 441         _have_valid_pieces = false;
 442 }
 443
 444 list<ReferencedReelAsset>
 445 Player::get_reel_assets ()
 446 {
 447         list<ReferencedReelAsset> a;
 448
 449         BOOST_FOREACH (shared_ptr<Content> i, _playlist->content ()) {
 450                 shared_ptr<DCPContent> j = dynamic_pointer_cast<DCPContent> (i);
 451                 if (!j) {
 452                         continue;
 453                 }
 454
 455                 scoped_ptr<DCPDecoder> decoder;
 456                 try {
 457                         decoder.reset (new DCPDecoder (j, _film->log()));
 458                 } catch (...) {
 459                         return a;
 460                 }
 461
 462                 int64_t offset = 0;
 463                 BOOST_FOREACH (shared_ptr<dcp::Reel> k, decoder->reels()) {
 464
 465                         DCPOMATIC_ASSERT (j->video_frame_rate ());
 466                         double const cfr = j->video_frame_rate().get();
 467                         Frame const trim_start = j->trim_start().frames_round (cfr);
 468                         Frame const trim_end = j->trim_end().frames_round (cfr);
 469                         int const ffr = _film->video_frame_rate ();
 470
 471                         DCPTime const from = i->position() + DCPTime::from_frames (offset, _film->video_frame_rate());
 472                         if (j->reference_video ()) {
 473                                 shared_ptr<dcp::ReelAsset> ra = k->main_picture ();
 474                                 DCPOMATIC_ASSERT (ra);
 475                                 ra->set_entry_point (ra->entry_point() + trim_start);
 476                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 477                                 a.push_back (
 478                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 479                                         );
 480                         }
 481
 482                         if (j->reference_audio ()) {
 483                                 shared_ptr<dcp::ReelAsset> ra = k->main_sound ();
 484                                 DCPOMATIC_ASSERT (ra);
 485                                 ra->set_entry_point (ra->entry_point() + trim_start);
 486                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 487                                 a.push_back (
 488                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 489                                         );
 490                         }
 491
 492                         if (j->reference_subtitle ()) {
 493                                 shared_ptr<dcp::ReelAsset> ra = k->main_subtitle ();
 494                                 DCPOMATIC_ASSERT (ra);
 495                                 ra->set_entry_point (ra->entry_point() + trim_start);
 496                                 ra->set_duration (ra->duration() - trim_start - trim_end);
 497                                 a.push_back (
 498                                         ReferencedReelAsset (ra, DCPTimePeriod (from, from + DCPTime::from_frames (ra->duration(), ffr)))
 499                                         );
 500                         }
 501
 502                         /* Assume that main picture duration is the length of the reel */
 503                         offset += k->main_picture()->duration ();
 504                 }
 505         }
 506
 507         return a;
 508 }
 509
 510 bool
 511 Player::pass ()
 512 {
 513         if (!_have_valid_pieces) {
 514                 setup_pieces ();
 515         }
 516
 517         shared_ptr<Piece> earliest;
 518         DCPTime earliest_content;
 519
 520         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 521                 if (!i->done) {
 522                         DCPTime const t = i->content->position() + DCPTime (i->decoder->position(), i->frc);
 523                         if (!earliest || t < earliest_content) {
 524                                 earliest_content = t;
 525                                 earliest = i;
 526                         }
 527                 }
 528         }
 529
 530         if (earliest) {
 531                 earliest->done = earliest->decoder->pass ();
 532                 if (earliest->done && earliest->content->audio) {
 533                         /* Flush the Player audio system for this piece */
 534                         BOOST_FOREACH (AudioStreamPtr i, earliest->content->audio->streams()) {
 535                                 audio_flush (earliest, i);
 536                         }
 537                 }
 538         }
 539
 540         /* Fill towards the next thing that might happen (or the end of the playlist).  This is to fill gaps between content,
 541            NOT to fill gaps within content (the latter is done in ::video())
 542         */
 543         DCPTime fill_towards = earliest ? earliest_content : _playlist->length();
 544
 545         /* Work out where to fill video from */
 546         optional<DCPTime> video_fill_from;
 547         if (_last_video_time) {
 548                 /* Fill from the last video or seek time */
 549                 video_fill_from = _last_video_time;
 550         }
 551
 552         bool filled = false;
 553         /* Fill some black if we would emit before the earliest piece of content.  This is so we act like a phantom
 554            Piece which emits black in spaces (we only emit if we are the earliest thing)
 555         */
 556         if (video_fill_from && (!earliest || *video_fill_from < earliest_content) && ((fill_towards - *video_fill_from)) >= one_video_frame()) {
 557                 list<DCPTimePeriod> p = subtract(DCPTimePeriod(*video_fill_from, *video_fill_from + one_video_frame()), _no_video);
 558                 if (!p.empty ()) {
 559                         emit_video (black_player_video_frame(), p.front().from);
 560                         filled = true;
 561                 }
 562         } else if (_playlist->length() == DCPTime()) {
 563                 /* Special case of an empty Film; just give one black frame */
 564                 emit_video (black_player_video_frame(), DCPTime());
 565                 filled = true;
 566         }
 567
 568         optional<DCPTime> audio_fill_from;
 569         if (_last_audio_time) {
 570                 /* Fill from the last audio or seek time */
 571                 audio_fill_from = _last_audio_time;
 572         }
 573
 574         /* XXX: _no_audio */
 575         if (audio_fill_from && audio_fill_from < fill_towards) {
 576                 DCPTimePeriod period (*audio_fill_from, fill_towards);
 577                 if (period.duration() > one_video_frame()) {
 578                         period.to = period.from + one_video_frame();
 579                 }
 580                 list<DCPTimePeriod> p = subtract(period, _no_video);
 581                 if (!p.empty ()) {
 582                         fill_audio (period);
 583                 }
 584                 filled = true;
 585         }
 586
 587         /* Emit any audio that is ready */
 588
 589         DCPTime pull_to = _playlist->length ();
 590         for (map<AudioStreamPtr, StreamState>::const_iterator i = _stream_states.begin(); i != _stream_states.end(); ++i) {
 591                 if (!i->second.piece->done && i->second.last_push_end < pull_to) {
 592                         pull_to = i->second.last_push_end;
 593                 }
 594         }
 595
 596         list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_to);
 597         for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
 598                 if (_last_audio_time && i->second < *_last_audio_time) {
 599                         /* There has been an accurate seek and we have received some audio before the seek time;
 600                            discard it.
 601                         */
 602                         pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (i->first, i->second, *_last_audio_time);
 603                         if (!cut.first) {
 604                                 continue;
 605                         }
 606                         *i = cut;
 607                 }
 608
 609                 if (_last_audio_time) {
 610                         fill_audio (DCPTimePeriod (*_last_audio_time, i->second));
 611                 }
 612
 613                 emit_audio (i->first, i->second);
 614         }
 615
 616         return !earliest && !filled;
 617 }
 618
 619 optional<PositionImage>
 620 Player::subtitles_for_frame (DCPTime time) const
 621 {
 622         list<PositionImage> subtitles;
 623
 624         BOOST_FOREACH (PlayerSubtitles i, _active_subtitles.get_burnt (time, _always_burn_subtitles)) {
 625
 626                 /* Image subtitles */
 627                 list<PositionImage> c = transform_image_subtitles (i.image);
 628                 copy (c.begin(), c.end(), back_inserter (subtitles));
 629
 630                 /* Text subtitles (rendered to an image) */
 631                 if (!i.text.empty ()) {
 632                         list<PositionImage> s = render_subtitles (i.text, i.fonts, _video_container_size, time);
 633                         copy (s.begin(), s.end(), back_inserter (subtitles));
 634                 }
 635         }
 636
 637         if (subtitles.empty ()) {
 638                 return optional<PositionImage> ();
 639         }
 640
 641         return merge (subtitles);
 642 }
 643
 644 void
 645 Player::video (weak_ptr<Piece> wp, ContentVideo video)
 646 {
 647         shared_ptr<Piece> piece = wp.lock ();
 648         if (!piece) {
 649                 return;
 650         }
 651
 652         FrameRateChange frc(piece->content->active_video_frame_rate(), _film->video_frame_rate());
 653         if (frc.skip && (video.frame % 2) == 1) {
 654                 return;
 655         }
 656
 657         /* Time and period of the frame we will emit */
 658         DCPTime const time = content_video_to_dcp (piece, video.frame);
 659         DCPTimePeriod const period (time, time + one_video_frame());
 660
 661         /* Discard if it's outside the content's period or if it's before the last accurate seek */
 662         if (
 663                 time < piece->content->position() ||
 664                 time >= piece->content->end() ||
 665                 (_last_video_time && time < *_last_video_time)) {
 666                 return;
 667         }
 668
 669         /* Fill gaps that we discover now that we have some video which needs to be emitted */
 670
 671         optional<DCPTime> fill_to;
 672         if (_last_video_time) {
 673                 fill_to = _last_video_time;
 674         }
 675
 676         if (fill_to) {
 677                 /* XXX: this may not work for 3D */
 678                 BOOST_FOREACH (DCPTimePeriod i, subtract(DCPTimePeriod (*fill_to, time), _no_video)) {
 679                         for (DCPTime j = i.from; j < i.to; j += one_video_frame()) {
 680                                 LastVideoMap::const_iterator k = _last_video.find (wp);
 681                                 if (k != _last_video.end ()) {
 682                                         emit_video (k->second, j);
 683                                 } else {
 684                                         emit_video (black_player_video_frame(), j);
 685                                 }
 686                         }
 687                 }
 688         }
 689
 690         _last_video[wp].reset (
 691                 new PlayerVideo (
 692                         video.image,
 693                         piece->content->video->crop (),
 694                         piece->content->video->fade (video.frame),
 695                         piece->content->video->scale().size (
 696                                 piece->content->video, _video_container_size, _film->frame_size ()
 697                                 ),
 698                         _video_container_size,
 699                         video.eyes,
 700                         video.part,
 701                         piece->content->video->colour_conversion ()
 702                         )
 703                 );
 704
 705         emit_video (_last_video[wp], time);
 706 }
 707
 708 void
 709 Player::audio_flush (shared_ptr<Piece> piece, AudioStreamPtr stream)
 710 {
 711         shared_ptr<AudioContent> content = piece->content->audio;
 712         DCPOMATIC_ASSERT (content);
 713
 714         shared_ptr<Resampler> r = resampler (content, stream, false);
 715         if (!r) {
 716                 return;
 717         }
 718
 719         pair<shared_ptr<const AudioBuffers>, Frame> ro = r->flush ();
 720         if (ro.first->frames() == 0) {
 721                 return;
 722         }
 723
 724         ContentAudio content_audio;
 725         content_audio.audio = ro.first;
 726         content_audio.frame = ro.second;
 727
 728         /* Compute time in the DCP */
 729         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame) + DCPTime::from_seconds (content->delay() / 1000.0);
 730
 731         audio_transform (content, stream, content_audio, time);
 732 }
 733
 734 /** Do our common processing on some audio */
 735 void
 736 Player::audio_transform (shared_ptr<AudioContent> content, AudioStreamPtr stream, ContentAudio content_audio, DCPTime time)
 737 {
 738         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 739
 740         /* Gain */
 741
 742         if (content->gain() != 0) {
 743                 shared_ptr<AudioBuffers> gain (new AudioBuffers (content_audio.audio));
 744                 gain->apply_gain (content->gain ());
 745                 content_audio.audio = gain;
 746         }
 747
 748         /* Remap */
 749
 750         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), content_audio.audio->frames()));
 751         dcp_mapped->make_silent ();
 752
 753         AudioMapping map = stream->mapping ();
 754         for (int i = 0; i < map.input_channels(); ++i) {
 755                 for (int j = 0; j < dcp_mapped->channels(); ++j) {
 756                         if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 757                                 dcp_mapped->accumulate_channel (
 758                                         content_audio.audio.get(),
 759                                         i,
 760                                         static_cast<dcp::Channel> (j),
 761                                         map.get (i, static_cast<dcp::Channel> (j))
 762                                         );
 763                         }
 764                 }
 765         }
 766
 767         content_audio.audio = dcp_mapped;
 768
 769         /* Process */
 770
 771         if (_audio_processor) {
 772                 content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 773         }
 774
 775         /* Pad any gap which may be caused by audio delay */
 776
 777         if (_last_audio_time) {
 778                 fill_audio (DCPTimePeriod (*_last_audio_time, time));
 779         }
 780
 781         /* Push */
 782
 783         _audio_merger.push (content_audio.audio, time);
 784         DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
 785         _stream_states[stream].last_push_end = time + DCPTime::from_frames (content_audio.audio->frames(), _film->audio_frame_rate());
 786 }
 787
 788 void
 789 Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 790 {
 791         DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
 792
 793         shared_ptr<Piece> piece = wp.lock ();
 794         if (!piece) {
 795                 return;
 796         }
 797
 798         shared_ptr<AudioContent> content = piece->content->audio;
 799         DCPOMATIC_ASSERT (content);
 800
 801         /* Resample */
 802         if (stream->frame_rate() != content->resampled_frame_rate()) {
 803                 shared_ptr<Resampler> r = resampler (content, stream, true);
 804                 pair<shared_ptr<const AudioBuffers>, Frame> ro = r->run (content_audio.audio, content_audio.frame);
 805                 if (ro.first->frames() == 0) {
 806                         return;
 807                 }
 808                 content_audio.audio = ro.first;
 809                 content_audio.frame = ro.second;
 810         }
 811
 812         /* Compute time in the DCP */
 813         DCPTime time = resampled_audio_to_dcp (piece, content_audio.frame) + DCPTime::from_seconds (content->delay() / 1000.0);
 814         /* And the end of this block in the DCP */
 815         DCPTime end = time + DCPTime::from_frames(content_audio.audio->frames(), content->resampled_frame_rate());
 816
 817         /* Remove anything that comes before the start or after the end of the content */
 818         if (time < piece->content->position()) {
 819                 pair<shared_ptr<AudioBuffers>, DCPTime> cut = discard_audio (content_audio.audio, time, piece->content->position());
 820                 if (!cut.first) {
 821                         /* This audio is entirely discarded */
 822                         return;
 823                 }
 824                 content_audio.audio = cut.first;
 825                 time = cut.second;
 826         } else if (time > piece->content->end()) {
 827                 /* Discard it all */
 828                 return;
 829         } else if (end > piece->content->end()) {
 830                 Frame const remaining_frames = DCPTime(piece->content->end() - time).frames_round(_film->audio_frame_rate());
 831                 if (remaining_frames == 0) {
 832                         return;
 833                 }
 834                 shared_ptr<AudioBuffers> cut (new AudioBuffers (content_audio.audio->channels(), remaining_frames));
 835                 cut->copy_from (content_audio.audio.get(), remaining_frames, 0, 0);
 836                 content_audio.audio = cut;
 837         }
 838
 839         audio_transform (content, stream, content_audio, time);
 840 }
 841
 842 void
 843 Player::image_subtitle_start (weak_ptr<Piece> wp, ContentImageSubtitle subtitle)
 844 {
 845         shared_ptr<Piece> piece = wp.lock ();
 846         if (!piece) {
 847                 return;
 848         }
 849
 850         /* Apply content's subtitle offsets */
 851         subtitle.sub.rectangle.x += piece->content->subtitle->x_offset ();
 852         subtitle.sub.rectangle.y += piece->content->subtitle->y_offset ();
 853
 854         /* Apply content's subtitle scale */
 855         subtitle.sub.rectangle.width *= piece->content->subtitle->x_scale ();
 856         subtitle.sub.rectangle.height *= piece->content->subtitle->y_scale ();
 857
 858         /* Apply a corrective translation to keep the subtitle centred after that scale */
 859         subtitle.sub.rectangle.x -= subtitle.sub.rectangle.width * (piece->content->subtitle->x_scale() - 1);
 860         subtitle.sub.rectangle.y -= subtitle.sub.rectangle.height * (piece->content->subtitle->y_scale() - 1);
 861
 862         PlayerSubtitles ps;
 863         ps.image.push_back (subtitle.sub);
 864         DCPTime from (content_time_to_dcp (piece, subtitle.from()));
 865
 866         _active_subtitles.add_from (wp, ps, from);
 867 }
 868
 869 void
 870 Player::text_subtitle_start (weak_ptr<Piece> wp, ContentTextSubtitle subtitle)
 871 {
 872         shared_ptr<Piece> piece = wp.lock ();
 873         if (!piece) {
 874                 return;
 875         }
 876
 877         PlayerSubtitles ps;
 878         DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
 879
 880         BOOST_FOREACH (dcp::SubtitleString s, subtitle.subs) {
 881                 s.set_h_position (s.h_position() + piece->content->subtitle->x_offset ());
 882                 s.set_v_position (s.v_position() + piece->content->subtitle->y_offset ());
 883                 float const xs = piece->content->subtitle->x_scale();
 884                 float const ys = piece->content->subtitle->y_scale();
 885                 float size = s.size();
 886
 887                 /* Adjust size to express the common part of the scaling;
 888                    e.g. if xs = ys = 0.5 we scale size by 2.
 889                 */
 890                 if (xs > 1e-5 && ys > 1e-5) {
 891                         size *= 1 / min (1 / xs, 1 / ys);
 892                 }
 893                 s.set_size (size);
 894
 895                 /* Then express aspect ratio changes */
 896                 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
 897                         s.set_aspect_adjust (xs / ys);
 898                 }
 899
 900                 s.set_in (dcp::Time(from.seconds(), 1000));
 901                 ps.text.push_back (SubtitleString (s, piece->content->subtitle->outline_width()));
 902                 ps.add_fonts (piece->content->subtitle->fonts ());
 903         }
 904
 905         _active_subtitles.add_from (wp, ps, from);
 906 }
 907
 908 void
 909 Player::subtitle_stop (weak_ptr<Piece> wp, ContentTime to)
 910 {
 911         if (!_active_subtitles.have (wp)) {
 912                 return;
 913         }
 914
 915         shared_ptr<Piece> piece = wp.lock ();
 916         if (!piece) {
 917                 return;
 918         }
 919
 920         DCPTime const dcp_to = content_time_to_dcp (piece, to);
 921
 922         pair<PlayerSubtitles, DCPTime> from = _active_subtitles.add_to (wp, dcp_to);
 923
 924         if (piece->content->subtitle->use() && !_always_burn_subtitles && !piece->content->subtitle->burn()) {
 925                 Subtitle (from.first, DCPTimePeriod (from.second, dcp_to));
 926         }
 927 }
 928
 929 void
 930 Player::seek (DCPTime time, bool accurate)
 931 {
 932         if (_audio_processor) {
 933                 _audio_processor->flush ();
 934         }
 935
 936         for (ResamplerMap::iterator i = _resamplers.begin(); i != _resamplers.end(); ++i) {
 937                 i->second->flush ();
 938                 i->second->reset ();
 939         }
 940
 941         _audio_merger.clear ();
 942         _active_subtitles.clear ();
 943
 944         BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
 945                 if (time < i->content->position()) {
 946                         /* Before; seek to 0 */
 947                         i->decoder->seek (ContentTime(), accurate);
 948                         i->done = false;
 949                 } else if (i->content->position() <= time && time < i->content->end()) {
 950                         /* During; seek to position */
 951                         i->decoder->seek (dcp_to_content_time (i, time), accurate);
 952                         i->done = false;
 953                 } else {
 954                         /* After; this piece is done */
 955                         i->done = true;
 956                 }
 957         }
 958
 959         if (accurate) {
 960                 _last_video_time = time;
 961                 _last_audio_time = time;
 962         } else {
 963                 _last_video_time = optional<DCPTime>();
 964                 _last_audio_time = optional<DCPTime>();
 965         }
 966 }
 967
 968 shared_ptr<Resampler>
 969 Player::resampler (shared_ptr<const AudioContent> content, AudioStreamPtr stream, bool create)
 970 {
 971         ResamplerMap::const_iterator i = _resamplers.find (make_pair (content, stream));
 972         if (i != _resamplers.end ()) {
 973                 return i->second;
 974         }
 975
 976         if (!create) {
 977                 return shared_ptr<Resampler> ();
 978         }
 979
 980         LOG_GENERAL (
 981                 "Creating new resampler from %1 to %2 with %3 channels",
 982                 stream->frame_rate(),
 983                 content->resampled_frame_rate(),
 984                 stream->channels()
 985                 );
 986
 987         shared_ptr<Resampler> r (
 988                 new Resampler (stream->frame_rate(), content->resampled_frame_rate(), stream->channels())
 989                 );
 990
 991         _resamplers[make_pair(content, stream)] = r;
 992         return r;
 993 }
 994
 995 void
 996 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
 997 {
 998         optional<PositionImage> subtitles = subtitles_for_frame (time);
 999         if (subtitles) {
1000                 pv->set_subtitle (subtitles.get ());
1001         }
1002
1003         Video (pv, time);
1004
1005         if (pv->eyes() == EYES_BOTH || pv->eyes() == EYES_RIGHT) {
1006                 _last_video_time = time + one_video_frame();
1007                 _active_subtitles.clear_before (time);
1008         }
1009 }
1010
1011 void
1012 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1013 {
1014         Audio (data, time);
1015         _last_audio_time = time + DCPTime::from_frames (data->frames(), _film->audio_frame_rate ());
1016 }
1017
1018 void
1019 Player::fill_audio (DCPTimePeriod period)
1020 {
1021         BOOST_FOREACH (DCPTimePeriod i, subtract(period, _no_audio)) {
1022                 DCPTime t = i.from;
1023                 while (t < i.to) {
1024                         DCPTime block = min (DCPTime::from_seconds (0.5), i.to - t);
1025                         Frame const samples = block.frames_round(_film->audio_frame_rate());
1026                         if (samples) {
1027                                 shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
1028                                 silence->make_silent ();
1029                                 emit_audio (silence, t);
1030                         }
1031                         t += block;
1032                 }
1033         }
1034 }
1035
1036 DCPTime
1037 Player::one_video_frame () const
1038 {
1039         return DCPTime::from_frames (1, _film->video_frame_rate ());
1040 }
1041
1042 pair<shared_ptr<AudioBuffers>, DCPTime>
1043 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1044 {
1045         DCPTime const discard_time = discard_to - time;
1046         Frame const discard_frames = discard_time.frames_round(_film->audio_frame_rate());
1047         Frame remaining_frames = audio->frames() - discard_frames;
1048         if (remaining_frames <= 0) {
1049                 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1050         }
1051         shared_ptr<AudioBuffers> cut (new AudioBuffers (audio->channels(), remaining_frames));
1052         cut->copy_from (audio.get(), remaining_frames, discard_frames, 0);
1053         return make_pair(cut, time + discard_time);
1054 }