src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "ratio.h"
  34 #include "log.h"
  35 #include "scaler.h"
  36
  37 using std::list;
  38 using std::cout;
  39 using std::min;
  40 using std::max;
  41 using std::vector;
  42 using std::pair;
  43 using std::map;
  44 using boost::shared_ptr;
  45 using boost::weak_ptr;
  46 using boost::dynamic_pointer_cast;
  47 using boost::optional;
  48
  49 class Piece
  50 {
  51 public:
  52         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d, FrameRateChange f)
  53                 : content (c)
  54                 , decoder (d)
  55                 , frc (f)
  56         {}
  57
  58         shared_ptr<Content> content;
  59         shared_ptr<Decoder> decoder;
  60         FrameRateChange frc;
  61 };
  62
  63 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  64         : _film (f)
  65         , _playlist (p)
  66         , _video (true)
  67         , _audio (true)
  68         , _have_valid_pieces (false)
  69         , _video_position (0)
  70         , _audio_position (0)
  71         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  72         , _last_emit_was_black (false)
  73         , _just_did_inaccurate_seek (false)
  74         , _approximate_size (false)
  75 {
  76         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  77         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  78         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  79         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
  80 }
  81
  82 void
  83 Player::disable_video ()
  84 {
  85         _video = false;
  86 }
  87
  88 void
  89 Player::disable_audio ()
  90 {
  91         _audio = false;
  92 }
  93
  94 bool
  95 Player::pass ()
  96 {
  97         if (!_have_valid_pieces) {
  98                 setup_pieces ();
  99         }
 100
 101         /* Interrogate all our pieces to find the one with the earliest decoded data */
 102
 103         shared_ptr<Piece> earliest_piece;
 104         shared_ptr<Decoded> earliest_decoded;
 105         DCPTime earliest_time = TIME_MAX;
 106         DCPTime earliest_audio = TIME_MAX;
 107
 108         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 109
 110                 shared_ptr<Decoded> dec = (*i)->decoder->peek ();
 111
 112                 if (dec) {
 113                         dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position());
 114                 }
 115
 116                 if (dec && dec->dcp_time < earliest_time) {
 117                         earliest_piece = *i;
 118                         earliest_decoded = dec;
 119                         earliest_time = dec->dcp_time;
 120                 }
 121
 122                 if (dynamic_pointer_cast<DecodedAudio> (dec) && dec->dcp_time < earliest_audio) {
 123                         earliest_audio = dec->dcp_time;
 124                 }
 125         }
 126
 127         if (!earliest_piece) {
 128                 flush ();
 129                 return true;
 130         }
 131
 132         if (earliest_audio != TIME_MAX) {
 133                 TimedAudioBuffers<DCPTime> tb = _audio_merger.pull (earliest_audio);
 134                 Audio (tb.audio, tb.time);
 135                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 136         }
 137
 138         /* Emit the earliest thing */
 139
 140         shared_ptr<DecodedVideo> dv = dynamic_pointer_cast<DecodedVideo> (earliest_decoded);
 141         shared_ptr<DecodedAudio> da = dynamic_pointer_cast<DecodedAudio> (earliest_decoded);
 142         shared_ptr<DecodedSubtitle> ds = dynamic_pointer_cast<DecodedSubtitle> (earliest_decoded);
 143
 144         if (dv && _video) {
 145                 DCPTime const half_frame = TIME_HZ / (2 * _film->video_frame_rate ());
 146                 if (!_just_did_inaccurate_seek && earliest_time > (_video_position + half_frame)) {
 147
 148                         /* See if we're inside some video content */
 149                         list<shared_ptr<Piece> >::iterator i = _pieces.begin();
 150                         while (i != _pieces.end() && ((*i)->content->position() >= _video_position || _video_position >= (*i)->content->end())) {
 151                                 ++i;
 152                         }
 153
 154                         if (i == _pieces.end() || !_last_incoming_video.video || !_have_valid_pieces) {
 155                                 /* We're outside all video content */
 156                                 emit_black ();
 157                         } else {
 158                                 _last_incoming_video.video->dcp_time = _video_position;
 159                                 emit_video (_last_incoming_video.weak_piece, _last_incoming_video.video);
 160                         }
 161
 162                 } else {
 163                         if (
 164                                 dv->dcp_time >= _video_position &&
 165                                 !earliest_piece->content->trimmed (dv->dcp_time - earliest_piece->content->position ())
 166                                 ) {
 167
 168                                 emit_video (earliest_piece, dv);
 169                         }
 170
 171                         earliest_piece->decoder->get ();
 172                 }
 173
 174         } else if (da && _audio) {
 175                 if (!_just_did_inaccurate_seek && earliest_time > _audio_position) {
 176                         emit_silence (earliest_time - _audio_position);
 177                 } else {
 178                         emit_audio (earliest_piece, da);
 179                         earliest_piece->decoder->get ();
 180                 }
 181         } else if (ds && _video) {
 182                 _in_subtitle.piece = earliest_piece;
 183                 _in_subtitle.subtitle = ds;
 184                 update_subtitle ();
 185                 earliest_piece->decoder->get ();
 186         }
 187
 188         _just_did_inaccurate_seek = false;
 189
 190         return false;
 191 }
 192
 193 void
 194 Player::emit_video (weak_ptr<Piece> weak_piece, shared_ptr<DecodedVideo> video)
 195 {
 196         /* Keep a note of what came in so that we can repeat it if required */
 197         _last_incoming_video.weak_piece = weak_piece;
 198         _last_incoming_video.video = video;
 199
 200         shared_ptr<Piece> piece = weak_piece.lock ();
 201         if (!piece) {
 202                 return;
 203         }
 204
 205         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 206         assert (content);
 207
 208         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
 209
 210         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
 211         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
 212         if (_approximate_size) {
 213                 image_size.width &= ~3;
 214                 image_size.height &= ~3;
 215         }
 216
 217         shared_ptr<PlayerImage> pi (
 218                 new PlayerImage (
 219                         video->image,
 220                         content->crop(),
 221                         image_size,
 222                         _video_container_size,
 223                         _film->scaler()
 224                         )
 225                 );
 226
 227         if (
 228                 _film->with_subtitles () &&
 229                 _out_subtitle.subtitle->image &&
 230                 video->dcp_time >= _out_subtitle.subtitle->dcp_time && video->dcp_time <= _out_subtitle.subtitle->dcp_time_to
 231                 ) {
 232
 233                 Position<int> const container_offset (
 234                         (_video_container_size.width - image_size.width) / 2,
 235                         (_video_container_size.height - image_size.width) / 2
 236                         );
 237
 238                 pi->set_subtitle (_out_subtitle.subtitle->image, _out_subtitle.position + container_offset);
 239         }
 240
 241 #ifdef DCPOMATIC_DEBUG
 242         _last_video = piece->content;
 243 #endif
 244
 245         Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time);
 246
 247         _last_emit_was_black = false;
 248         _video_position = rint (video->dcp_time + TIME_HZ / _film->video_frame_rate());
 249 }
 250
 251 void
 252 Player::emit_audio (weak_ptr<Piece> weak_piece, shared_ptr<DecodedAudio> audio)
 253 {
 254         shared_ptr<Piece> piece = weak_piece.lock ();
 255         if (!piece) {
 256                 return;
 257         }
 258
 259         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 260         assert (content);
 261
 262         /* Gain */
 263         if (content->audio_gain() != 0) {
 264                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio->data));
 265                 gain->apply_gain (content->audio_gain ());
 266                 audio->data = gain;
 267         }
 268
 269         if (content->trimmed (audio->dcp_time - content->position ())) {
 270                 return;
 271         }
 272
 273         /* Remap channels */
 274         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames()));
 275         dcp_mapped->make_silent ();
 276         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 277         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 278                 if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) {
 279                         dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second);
 280                 }
 281         }
 282
 283         audio->data = dcp_mapped;
 284
 285         /* Delay */
 286         audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
 287         if (audio->dcp_time < 0) {
 288                 int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
 289                 if (frames >= audio->data->frames ()) {
 290                         return;
 291                 }
 292
 293                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames));
 294                 trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0);
 295
 296                 audio->data = trimmed;
 297                 audio->dcp_time = 0;
 298         }
 299
 300         _audio_merger.push (audio->data, audio->dcp_time);
 301 }
 302
 303 void
 304 Player::flush ()
 305 {
 306         TimedAudioBuffers<DCPTime> tb = _audio_merger.flush ();
 307         if (tb.audio) {
 308                 Audio (tb.audio, tb.time);
 309                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 310         }
 311
 312         while (_video_position < _audio_position) {
 313                 emit_black ();
 314         }
 315
 316         while (_audio_position < _video_position) {
 317                 emit_silence (_video_position - _audio_position);
 318         }
 319
 320 }
 321
 322 /** Seek so that the next pass() will yield (approximately) the requested frame.
 323  *  Pass accurate = true to try harder to get close to the request.
 324  *  @return true on error
 325  */
 326 void
 327 Player::seek (DCPTime t, bool accurate)
 328 {
 329         if (!_have_valid_pieces) {
 330                 setup_pieces ();
 331         }
 332
 333         if (_pieces.empty ()) {
 334                 return;
 335         }
 336
 337         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 338                 /* s is the offset of t from the start position of this content */
 339                 DCPTime s = t - (*i)->content->position ();
 340                 s = max (static_cast<DCPTime> (0), s);
 341                 s = min ((*i)->content->length_after_trim(), s);
 342
 343                 /* Convert this to the content time */
 344                 ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start ();
 345
 346                 /* And seek the decoder */
 347                 (*i)->decoder->seek (ct, accurate);
 348         }
 349
 350         _video_position = time_round_up (t, TIME_HZ / _film->video_frame_rate());
 351         _audio_position = time_round_up (t, TIME_HZ / _film->audio_frame_rate());
 352
 353         _audio_merger.clear (_audio_position);
 354
 355         if (!accurate) {
 356                 /* We just did an inaccurate seek, so it's likely that the next thing seen
 357                    out of pass() will be a fair distance from _{video,audio}_position.  Setting
 358                    this flag stops pass() from trying to fix that: we assume that if it
 359                    was an inaccurate seek then the caller does not care too much about
 360                    inserting black/silence to keep the time tidy.
 361                 */
 362                 _just_did_inaccurate_seek = true;
 363         }
 364 }
 365
 366 void
 367 Player::setup_pieces ()
 368 {
 369         list<shared_ptr<Piece> > old_pieces = _pieces;
 370         _pieces.clear ();
 371
 372         ContentList content = _playlist->content ();
 373
 374         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 375
 376                 shared_ptr<Decoder> decoder;
 377                 optional<FrameRateChange> frc;
 378
 379                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 380                 if (fc) {
 381                         decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio));
 382                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 383                 }
 384
 385                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 386                 if (ic) {
 387                         /* See if we can re-use an old ImageDecoder */
 388                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 389                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 390                                 if (imd && imd->content() == ic) {
 391                                         decoder = imd;
 392                                 }
 393                         }
 394
 395                         if (!decoder) {
 396                                 decoder.reset (new ImageDecoder (_film, ic));
 397                         }
 398
 399                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 400                 }
 401
 402                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 403                 if (sc) {
 404                         decoder.reset (new SndfileDecoder (_film, sc));
 405
 406                         /* Working out the frc for this content is a bit tricky: what if it overlaps
 407                            two pieces of video content with different frame rates?  For now, use
 408                            the one with the best overlap.
 409                         */
 410
 411                         DCPTime best_overlap_t = 0;
 412                         shared_ptr<VideoContent> best_overlap;
 413                         for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
 414                                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
 415                                 if (!vc) {
 416                                         continue;
 417                                 }
 418
 419                                 DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end());
 420                                 if (overlap > best_overlap_t) {
 421                                         best_overlap = vc;
 422                                         best_overlap_t = overlap;
 423                                 }
 424                         }
 425
 426                         if (best_overlap) {
 427                                 frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 428                         } else {
 429                                 /* No video overlap; e.g. if the DCP is just audio */
 430                                 frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 431                         }
 432                 }
 433
 434                 decoder->seek ((*i)->trim_start (), true);
 435
 436                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 437         }
 438
 439         _have_valid_pieces = true;
 440 }
 441
 442 void
 443 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 444 {
 445         shared_ptr<Content> c = w.lock ();
 446         if (!c) {
 447                 return;
 448         }
 449
 450         if (
 451                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 452                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 453                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 454                 ) {
 455
 456                 _have_valid_pieces = false;
 457                 Changed (frequent);
 458
 459         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
 460
 461                 update_subtitle ();
 462                 Changed (frequent);
 463
 464         } else if (property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO) {
 465
 466                 Changed (frequent);
 467
 468         } else if (property == ContentProperty::PATH) {
 469
 470                 Changed (frequent);
 471         }
 472 }
 473
 474 void
 475 Player::playlist_changed ()
 476 {
 477         _have_valid_pieces = false;
 478         Changed (false);
 479 }
 480
 481 void
 482 Player::set_video_container_size (libdcp::Size s)
 483 {
 484         _video_container_size = s;
 485
 486         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 487         im->make_black ();
 488
 489         _black_frame.reset (
 490                 new PlayerImage (
 491                         im,
 492                         Crop(),
 493                         _video_container_size,
 494                         _video_container_size,
 495                         Scaler::from_id ("bicubic")
 496                         )
 497                 );
 498 }
 499
 500 void
 501 Player::emit_black ()
 502 {
 503 #ifdef DCPOMATIC_DEBUG
 504         _last_video.reset ();
 505 #endif
 506
 507         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
 508         _video_position += _film->video_frames_to_time (1);
 509         _last_emit_was_black = true;
 510 }
 511
 512 void
 513 Player::emit_silence (DCPTime most)
 514 {
 515         if (most == 0) {
 516                 return;
 517         }
 518
 519         DCPTime t = min (most, TIME_HZ / 2);
 520         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ));
 521         silence->make_silent ();
 522         Audio (silence, _audio_position);
 523
 524         _audio_position += t;
 525 }
 526
 527 void
 528 Player::film_changed (Film::Property p)
 529 {
 530         /* Here we should notice Film properties that affect our output, and
 531            alert listeners that our output now would be different to how it was
 532            last time we were run.
 533         */
 534
 535         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 536                 Changed (false);
 537         }
 538 }
 539
 540 void
 541 Player::update_subtitle ()
 542 {
 543         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
 544         if (!piece) {
 545                 return;
 546         }
 547
 548         if (!_in_subtitle.subtitle->image) {
 549                 _out_subtitle.subtitle->image.reset ();
 550                 return;
 551         }
 552
 553         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 554         assert (sc);
 555
 556         dcpomatic::Rect<double> in_rect = _in_subtitle.subtitle->rect;
 557         libdcp::Size scaled_size;
 558
 559         in_rect.y += sc->subtitle_offset ();
 560
 561         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 562         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
 563         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
 564
 565         /* Then we need a corrective translation, consisting of two parts:
 566          *
 567          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 568          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 569          *
 570          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 571          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 572          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 573          *
 574          * Combining these two translations gives these expressions.
 575          */
 576
 577         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
 578         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
 579
 580         _out_subtitle.subtitle->image = _in_subtitle.subtitle->image->scale (
 581                 scaled_size,
 582                 Scaler::from_id ("bicubic"),
 583                 _in_subtitle.subtitle->image->pixel_format (),
 584                 true
 585                 );
 586
 587         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
 588         _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time;
 589 }
 590
 591 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 592  *  @return false if this could not be done.
 593  */
 594 bool
 595 Player::repeat_last_video ()
 596 {
 597         if (!_last_incoming_video.video || !_have_valid_pieces) {
 598                 return false;
 599         }
 600
 601         emit_video (
 602                 _last_incoming_video.weak_piece,
 603                 _last_incoming_video.video
 604                 );
 605
 606         return true;
 607 }
 608
 609 void
 610 Player::set_approximate_size ()
 611 {
 612         _approximate_size = true;
 613 }
 614
 615
 616 PlayerImage::PlayerImage (
 617         shared_ptr<const Image> in,
 618         Crop crop,
 619         libdcp::Size inter_size,
 620         libdcp::Size out_size,
 621         Scaler const * scaler
 622         )
 623         : _in (in)
 624         , _crop (crop)
 625         , _inter_size (inter_size)
 626         , _out_size (out_size)
 627         , _scaler (scaler)
 628 {
 629
 630 }
 631
 632 void
 633 PlayerImage::set_subtitle (shared_ptr<const Image> image, Position<int> pos)
 634 {
 635         _subtitle_image = image;
 636         _subtitle_position = pos;
 637 }
 638
 639 shared_ptr<Image>
 640 PlayerImage::image (AVPixelFormat format, bool aligned)
 641 {
 642         shared_ptr<Image> out = _in->crop_scale_window (_crop, _inter_size, _out_size, _scaler, format, aligned);
 643
 644         Position<int> const container_offset ((_out_size.width - _inter_size.width) / 2, (_out_size.height - _inter_size.width) / 2);
 645
 646         if (_subtitle_image) {
 647                 out->alpha_blend (_subtitle_image, _subtitle_position);
 648         }
 649
 650         return out;
 651 }
 652