src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "still_image_decoder.h"
  26 #include "still_image_content.h"
  27 #include "moving_image_decoder.h"
  28 #include "moving_image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "playlist.h"
  33 #include "job.h"
  34 #include "image.h"
  35 #include "ratio.h"
  36 #include "resampler.h"
  37 #include "log.h"
  38 #include "scaler.h"
  39
  40 using std::list;
  41 using std::cout;
  42 using std::min;
  43 using std::max;
  44 using std::vector;
  45 using std::pair;
  46 using std::map;
  47 using boost::shared_ptr;
  48 using boost::weak_ptr;
  49 using boost::dynamic_pointer_cast;
  50
  51 //#define DEBUG_PLAYER 1
  52
  53 class Piece
  54 {
  55 public:
  56         Piece (shared_ptr<Content> c)
  57                 : content (c)
  58                 , video_position (c->start ())
  59                 , audio_position (c->start ())
  60         {}
  61
  62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  63                 : content (c)
  64                 , decoder (d)
  65                 , video_position (c->start ())
  66                 , audio_position (c->start ())
  67         {}
  68
  69         shared_ptr<Content> content;
  70         shared_ptr<Decoder> decoder;
  71         Time video_position;
  72         Time audio_position;
  73 };
  74
  75 #ifdef DEBUG_PLAYER
  76 std::ostream& operator<<(std::ostream& s, Piece const & p)
  77 {
  78         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
  79                 s << "\tffmpeg     ";
  80         } else if (dynamic_pointer_cast<StillImageContent> (p.content)) {
  81                 s << "\tstill image";
  82         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
  83                 s << "\tsndfile    ";
  84         }
  85
  86         s << " at " << p.content->start() << " until " << p.content->end();
  87
  88         return s;
  89 }
  90 #endif
  91
  92 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  93         : _film (f)
  94         , _playlist (p)
  95         , _video (true)
  96         , _audio (true)
  97         , _have_valid_pieces (false)
  98         , _video_position (0)
  99         , _audio_position (0)
 100         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
 101         , _last_emit_was_black (false)
 102 {
 103         _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 104         _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
 105         _film->Changed.connect (bind (&Player::film_changed, this, _1));
 106         set_video_container_size (_film->container()->size (_film->full_frame ()));
 107 }
 108
 109 void
 110 Player::disable_video ()
 111 {
 112         _video = false;
 113 }
 114
 115 void
 116 Player::disable_audio ()
 117 {
 118         _audio = false;
 119 }
 120
 121 bool
 122 Player::pass ()
 123 {
 124         if (!_have_valid_pieces) {
 125                 setup_pieces ();
 126                 _have_valid_pieces = true;
 127         }
 128
 129 #ifdef DEBUG_PLAYER
 130         cout << "= PASS\n";
 131 #endif
 132
 133         Time earliest_t = TIME_MAX;
 134         shared_ptr<Piece> earliest;
 135         enum {
 136                 VIDEO,
 137                 AUDIO
 138         } type = VIDEO;
 139
 140         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 141                 if ((*i)->decoder->done ()) {
 142                         continue;
 143                 }
 144
 145                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 146                         if ((*i)->video_position < earliest_t) {
 147                                 earliest_t = (*i)->video_position;
 148                                 earliest = *i;
 149                                 type = VIDEO;
 150                         }
 151                 }
 152
 153                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 154                         if ((*i)->audio_position < earliest_t) {
 155                                 earliest_t = (*i)->audio_position;
 156                                 earliest = *i;
 157                                 type = AUDIO;
 158                         }
 159                 }
 160         }
 161
 162         if (!earliest) {
 163 #ifdef DEBUG_PLAYER
 164                 cout << "no earliest piece.\n";
 165 #endif
 166
 167                 flush ();
 168                 return true;
 169         }
 170
 171         switch (type) {
 172         case VIDEO:
 173                 if (earliest_t > _video_position) {
 174 #ifdef DEBUG_PLAYER
 175                         cout << "no video here; emitting black frame (earliest=" << earliest_t << ", video_position=" << _video_position << ").\n";
 176 #endif
 177                         emit_black ();
 178                 } else {
 179 #ifdef DEBUG_PLAYER
 180                         cout << "Pass " << *earliest << "\n";
 181 #endif
 182                         earliest->decoder->pass ();
 183                 }
 184                 break;
 185
 186         case AUDIO:
 187                 if (earliest_t > _audio_position) {
 188 #ifdef DEBUG_PLAYER
 189                         cout << "no audio here; emitting silence.\n";
 190 #endif
 191                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 192                 } else {
 193 #ifdef DEBUG_PLAYER
 194                         cout << "Pass " << *earliest << "\n";
 195 #endif
 196                         earliest->decoder->pass ();
 197
 198                         if (earliest->decoder->done()) {
 199                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 200                                 assert (ac);
 201                                 shared_ptr<Resampler> re = resampler (ac, false);
 202                                 if (re) {
 203                                         shared_ptr<const AudioBuffers> b = re->flush ();
 204                                         if (b->frames ()) {
 205                                                 process_audio (earliest, b, ac->audio_length ());
 206                                         }
 207                                 }
 208                         }
 209
 210
 211                 }
 212
 213                 Time done_up_to = TIME_MAX;
 214                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 215                         if (dynamic_pointer_cast<AudioContent> ((*i)->content)) {
 216                                 done_up_to = min (done_up_to, (*i)->audio_position);
 217                         }
 218                 }
 219
 220                 TimedAudioBuffers<Time> tb = _audio_merger.pull (done_up_to);
 221                 Audio (tb.audio, tb.time);
 222                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 223                 break;
 224         }
 225
 226
 227
 228 #ifdef DEBUG_PLAYER
 229         cout << "\tpost pass " << _video_position << " " << _audio_position << "\n";
 230 #endif
 231
 232         return false;
 233 }
 234
 235 void
 236 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
 237 {
 238         shared_ptr<Piece> piece = weak_piece.lock ();
 239         if (!piece) {
 240                 return;
 241         }
 242
 243         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 244         assert (content);
 245
 246         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 247         if (frc.skip && (frame % 2) == 1) {
 248                 return;
 249         }
 250
 251         shared_ptr<Image> work_image = image->crop (content->crop(), true);
 252
 253         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
 254
 255         work_image = work_image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
 256
 257         Time time = content->start() + (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 258
 259         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
 260                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
 261         }
 262
 263         if (image_size != _video_container_size) {
 264                 assert (image_size.width <= _video_container_size.width);
 265                 assert (image_size.height <= _video_container_size.height);
 266                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 267                 im->make_black ();
 268                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 269                 work_image = im;
 270         }
 271
 272 #ifdef DCPOMATIC_DEBUG
 273         _last_video = piece->content;
 274 #endif
 275
 276         Video (work_image, eyes, same, time);
 277         time += TIME_HZ / _film->video_frame_rate();
 278
 279         if (frc.repeat) {
 280                 Video (work_image, eyes, true, time);
 281                 time += TIME_HZ / _film->video_frame_rate();
 282         }
 283
 284         _last_emit_was_black = false;
 285
 286         _video_position = piece->video_position = time;
 287 }
 288
 289 void
 290 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 291 {
 292         shared_ptr<Piece> piece = weak_piece.lock ();
 293         if (!piece) {
 294                 return;
 295         }
 296
 297         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 298         assert (content);
 299
 300         /* Resample */
 301         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 302                 shared_ptr<Resampler> r = resampler (content, true);
 303                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 304                 audio = ro.first;
 305                 frame = ro.second;
 306         }
 307
 308         /* Remap channels */
 309         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 310         dcp_mapped->make_silent ();
 311         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 312         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 313                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
 314                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 315                 }
 316         }
 317
 318         audio = dcp_mapped;
 319
 320         Time time = content->start()
 321                 + _film->audio_frames_to_time (frame)
 322                 + (content->audio_delay() * TIME_HZ / 1000);
 323
 324         /* We must cut off anything that comes before the start of all time */
 325         if (time < 0) {
 326                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 327                 if (frames >= audio->frames ()) {
 328                         return;
 329                 }
 330
 331                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 332                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 333
 334                 audio = trimmed;
 335                 time = 0;
 336         }
 337
 338         _audio_merger.push (audio, time);
 339         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 340 }
 341
 342 void
 343 Player::flush ()
 344 {
 345         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 346         if (tb.audio) {
 347                 Audio (tb.audio, tb.time);
 348                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 349         }
 350
 351         while (_video_position < _audio_position) {
 352                 emit_black ();
 353         }
 354
 355         while (_audio_position < _video_position) {
 356                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 357         }
 358
 359 }
 360
 361 /** Seek so that the next pass() will yield (approximately) the requested frame.
 362  *  Pass accurate = true to try harder to get close to the request.
 363  *  @return true on error
 364  */
 365 void
 366 Player::seek (Time t, bool accurate)
 367 {
 368         if (!_have_valid_pieces) {
 369                 setup_pieces ();
 370                 _have_valid_pieces = true;
 371         }
 372
 373         if (_pieces.empty ()) {
 374                 return;
 375         }
 376
 377         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 378                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 379                 if (!vc) {
 380                         continue;
 381                 }
 382
 383                 Time s = t - vc->start ();
 384                 s = max (static_cast<Time> (0), s);
 385                 s = min (vc->length(), s);
 386
 387                 (*i)->video_position = (*i)->audio_position = vc->start() + s;
 388
 389                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
 390                 /* Here we are converting from time (in the DCP) to a frame number in the content.
 391                    Hence we need to use the DCP's frame rate and the double/skip correction, not
 392                    the source's rate.
 393                 */
 394                 VideoContent::Frame f = s * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
 395                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 396         }
 397
 398         _video_position = _audio_position = t;
 399
 400         /* XXX: don't seek audio because we don't need to... */
 401 }
 402
 403 void
 404 Player::setup_pieces ()
 405 {
 406         list<shared_ptr<Piece> > old_pieces = _pieces;
 407
 408         _pieces.clear ();
 409
 410         ContentList content = _playlist->content ();
 411         sort (content.begin(), content.end(), ContentSorter ());
 412
 413         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 414
 415                 shared_ptr<Piece> piece (new Piece (*i));
 416
 417                 /* XXX: into content? */
 418
 419                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 420                 if (fc) {
 421                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 422
 423                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 424                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 425                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
 426
 427                         piece->decoder = fd;
 428                 }
 429
 430                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
 431                 if (ic) {
 432                         shared_ptr<StillImageDecoder> id;
 433
 434                         /* See if we can re-use an old StillImageDecoder */
 435                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 436                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
 437                                 if (imd && imd->content() == ic) {
 438                                         id = imd;
 439                                 }
 440                         }
 441
 442                         if (!id) {
 443                                 id.reset (new StillImageDecoder (_film, ic));
 444                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 445                         }
 446
 447                         piece->decoder = id;
 448                 }
 449
 450                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
 451                 if (mc) {
 452                         shared_ptr<MovingImageDecoder> md;
 453
 454                         if (!md) {
 455                                 md.reset (new MovingImageDecoder (_film, mc));
 456                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 457                         }
 458
 459                         piece->decoder = md;
 460                 }
 461
 462                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 463                 if (sc) {
 464                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 465                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 466
 467                         piece->decoder = sd;
 468                 }
 469
 470                 _pieces.push_back (piece);
 471         }
 472
 473 #ifdef DEBUG_PLAYER
 474         cout << "=== Player setup:\n";
 475         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 476                 cout << *(i->get()) << "\n";
 477         }
 478 #endif
 479 }
 480
 481 void
 482 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 483 {
 484         shared_ptr<Content> c = w.lock ();
 485         if (!c) {
 486                 return;
 487         }
 488
 489         if (
 490                 property == ContentProperty::START || property == ContentProperty::LENGTH ||
 491                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO
 492                 ) {
 493
 494                 _have_valid_pieces = false;
 495                 Changed (frequent);
 496
 497         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
 498                 update_subtitle ();
 499                 Changed (frequent);
 500         } else if (property == VideoContentProperty::VIDEO_FRAME_TYPE) {
 501                 Changed (frequent);
 502         }
 503 }
 504
 505 void
 506 Player::playlist_changed ()
 507 {
 508         _have_valid_pieces = false;
 509         Changed (false);
 510 }
 511
 512 void
 513 Player::set_video_container_size (libdcp::Size s)
 514 {
 515         _video_container_size = s;
 516         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 517         _black_frame->make_black ();
 518 }
 519
 520 shared_ptr<Resampler>
 521 Player::resampler (shared_ptr<AudioContent> c, bool create)
 522 {
 523         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 524         if (i != _resamplers.end ()) {
 525                 return i->second;
 526         }
 527
 528         if (!create) {
 529                 return shared_ptr<Resampler> ();
 530         }
 531
 532         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 533         _resamplers[c] = r;
 534         return r;
 535 }
 536
 537 void
 538 Player::emit_black ()
 539 {
 540 #ifdef DCPOMATIC_DEBUG
 541         _last_video.reset ();
 542 #endif
 543
 544         Video (_black_frame, EYES_BOTH, _last_emit_was_black, _video_position);
 545         _video_position += _film->video_frames_to_time (1);
 546         _last_emit_was_black = true;
 547 }
 548
 549 void
 550 Player::emit_silence (OutputAudioFrame most)
 551 {
 552         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 553         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 554         silence->make_silent ();
 555         Audio (silence, _audio_position);
 556         _audio_position += _film->audio_frames_to_time (N);
 557 }
 558
 559 void
 560 Player::film_changed (Film::Property p)
 561 {
 562         /* Here we should notice Film properties that affect our output, and
 563            alert listeners that our output now would be different to how it was
 564            last time we were run.
 565         */
 566
 567         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 568                 Changed (false);
 569         }
 570 }
 571
 572 void
 573 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 574 {
 575         _in_subtitle.piece = weak_piece;
 576         _in_subtitle.image = image;
 577         _in_subtitle.rect = rect;
 578         _in_subtitle.from = from;
 579         _in_subtitle.to = to;
 580
 581         update_subtitle ();
 582 }
 583
 584 void
 585 Player::update_subtitle ()
 586 {
 587         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
 588         if (!piece) {
 589                 return;
 590         }
 591
 592         if (!_in_subtitle.image) {
 593                 _out_subtitle.image.reset ();
 594                 return;
 595         }
 596
 597         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 598         assert (sc);
 599
 600         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
 601         libdcp::Size scaled_size;
 602
 603         in_rect.y += sc->subtitle_offset ();
 604
 605         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 606         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
 607         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
 608
 609         /* Then we need a corrective translation, consisting of two parts:
 610          *
 611          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 612          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 613          *
 614          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 615          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 616          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 617          *
 618          * Combining these two translations gives these expressions.
 619          */
 620
 621         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
 622         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
 623
 624         _out_subtitle.image = _in_subtitle.image->scale (libdcp::Size (scaled_size.width, scaled_size.height), Scaler::from_id ("bicubic"), true);
 625         _out_subtitle.from = _in_subtitle.from + piece->content->start ();
 626         _out_subtitle.to = _in_subtitle.to + piece->content->start ();
 627 }