src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "still_image_decoder.h"
  26 #include "still_image_content.h"
  27 #include "moving_image_decoder.h"
  28 #include "moving_image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "playlist.h"
  33 #include "job.h"
  34 #include "image.h"
  35 #include "ratio.h"
  36 #include "resampler.h"
  37 #include "log.h"
  38 #include "scaler.h"
  39
  40 using std::list;
  41 using std::cout;
  42 using std::min;
  43 using std::max;
  44 using std::vector;
  45 using std::pair;
  46 using std::map;
  47 using boost::shared_ptr;
  48 using boost::weak_ptr;
  49 using boost::dynamic_pointer_cast;
  50
  51 //#define DEBUG_PLAYER 1
  52
  53 class Piece
  54 {
  55 public:
  56         Piece (shared_ptr<Content> c)
  57                 : content (c)
  58                 , video_position (c->position ())
  59                 , audio_position (c->position ())
  60         {}
  61
  62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  63                 : content (c)
  64                 , decoder (d)
  65                 , video_position (c->position ())
  66                 , audio_position (c->position ())
  67         {}
  68
  69         shared_ptr<Content> content;
  70         shared_ptr<Decoder> decoder;
  71         Time video_position;
  72         Time audio_position;
  73 };
  74
  75 #ifdef DEBUG_PLAYER
  76 std::ostream& operator<<(std::ostream& s, Piece const & p)
  77 {
  78         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
  79                 s << "\tffmpeg     ";
  80         } else if (dynamic_pointer_cast<StillImageContent> (p.content)) {
  81                 s << "\tstill image";
  82         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
  83                 s << "\tsndfile    ";
  84         }
  85
  86         s << " at " << p.content->position() << " until " << p.content->end();
  87
  88         return s;
  89 }
  90 #endif
  91
  92 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  93         : _film (f)
  94         , _playlist (p)
  95         , _video (true)
  96         , _audio (true)
  97         , _have_valid_pieces (false)
  98         , _video_position (0)
  99         , _audio_position (0)
 100         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
 101         , _last_emit_was_black (false)
 102 {
 103         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 104         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
 105         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
 106         set_video_container_size (_film->container()->size (_film->full_frame ()));
 107 }
 108
 109 void
 110 Player::disable_video ()
 111 {
 112         _video = false;
 113 }
 114
 115 void
 116 Player::disable_audio ()
 117 {
 118         _audio = false;
 119 }
 120
 121 bool
 122 Player::pass ()
 123 {
 124         if (!_have_valid_pieces) {
 125                 setup_pieces ();
 126                 _have_valid_pieces = true;
 127         }
 128
 129 #ifdef DEBUG_PLAYER
 130         cout << "= PASS\n";
 131 #endif
 132
 133         Time earliest_t = TIME_MAX;
 134         shared_ptr<Piece> earliest;
 135         enum {
 136                 VIDEO,
 137                 AUDIO
 138         } type = VIDEO;
 139
 140         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 141                 if ((*i)->decoder->done ()) {
 142                         continue;
 143                 }
 144
 145                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 146                         if ((*i)->video_position < earliest_t) {
 147                                 earliest_t = (*i)->video_position;
 148                                 earliest = *i;
 149                                 type = VIDEO;
 150                         }
 151                 }
 152
 153                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 154                         if ((*i)->audio_position < earliest_t) {
 155                                 earliest_t = (*i)->audio_position;
 156                                 earliest = *i;
 157                                 type = AUDIO;
 158                         }
 159                 }
 160         }
 161
 162         if (!earliest) {
 163 #ifdef DEBUG_PLAYER
 164                 cout << "no earliest piece.\n";
 165 #endif
 166
 167                 flush ();
 168                 return true;
 169         }
 170
 171         switch (type) {
 172         case VIDEO:
 173                 if (earliest_t > _video_position) {
 174 #ifdef DEBUG_PLAYER
 175                         cout << "no video here; emitting black frame (earliest=" << earliest_t << ", video_position=" << _video_position << ").\n";
 176 #endif
 177                         emit_black ();
 178                 } else {
 179 #ifdef DEBUG_PLAYER
 180                         cout << "Pass video " << *earliest << "\n";
 181 #endif
 182                         earliest->decoder->pass ();
 183                 }
 184                 break;
 185
 186         case AUDIO:
 187                 if (earliest_t > _audio_position) {
 188 #ifdef DEBUG_PLAYER
 189                         cout << "no audio here (none until " << earliest_t << "); emitting silence.\n";
 190 #endif
 191                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 192                 } else {
 193 #ifdef DEBUG_PLAYER
 194                         cout << "Pass audio " << *earliest << "\n";
 195 #endif
 196                         earliest->decoder->pass ();
 197
 198                         if (earliest->decoder->done()) {
 199                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 200                                 assert (ac);
 201                                 shared_ptr<Resampler> re = resampler (ac, false);
 202                                 if (re) {
 203                                         shared_ptr<const AudioBuffers> b = re->flush ();
 204                                         if (b->frames ()) {
 205                                                 process_audio (earliest, b, ac->audio_length ());
 206                                         }
 207                                 }
 208                         }
 209                 }
 210                 break;
 211         }
 212
 213         if (_audio) {
 214                 Time audio_done_up_to = TIME_MAX;
 215                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 216                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 217                                 audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
 218                         }
 219                 }
 220
 221                 TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
 222                 Audio (tb.audio, tb.time);
 223                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 224         }
 225
 226 #ifdef DEBUG_PLAYER
 227         cout << "\tpost pass _video_position=" << _video_position << " _audio_position=" << _audio_position << "\n";
 228 #endif
 229
 230         return false;
 231 }
 232
 233 void
 234 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
 235 {
 236         shared_ptr<Piece> piece = weak_piece.lock ();
 237         if (!piece) {
 238                 return;
 239         }
 240
 241         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 242         assert (content);
 243
 244         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 245         if (frc.skip && (frame % 2) == 1) {
 246                 return;
 247         }
 248
 249         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 250         if (content->trimmed (relative_time)) {
 251                 return;
 252         }
 253
 254         shared_ptr<Image> work_image = image->crop (content->crop(), true);
 255
 256         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
 257
 258         work_image = work_image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
 259
 260         Time time = content->position() + relative_time - content->trim_start ();
 261
 262         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
 263                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
 264         }
 265
 266         if (image_size != _video_container_size) {
 267                 assert (image_size.width <= _video_container_size.width);
 268                 assert (image_size.height <= _video_container_size.height);
 269                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 270                 im->make_black ();
 271                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 272                 work_image = im;
 273         }
 274
 275 #ifdef DCPOMATIC_DEBUG
 276         _last_video = piece->content;
 277 #endif
 278
 279         Video (work_image, eyes, content->colour_conversion(), same, time);
 280         time += TIME_HZ / _film->video_frame_rate();
 281
 282         if (frc.repeat) {
 283                 Video (work_image, eyes, content->colour_conversion(), true, time);
 284                 time += TIME_HZ / _film->video_frame_rate();
 285         }
 286
 287         _last_emit_was_black = false;
 288
 289         _video_position = piece->video_position = time;
 290 }
 291
 292 void
 293 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 294 {
 295         shared_ptr<Piece> piece = weak_piece.lock ();
 296         if (!piece) {
 297                 return;
 298         }
 299
 300         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 301         assert (content);
 302
 303         /* Resample */
 304         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 305                 shared_ptr<Resampler> r = resampler (content, true);
 306                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 307                 audio = ro.first;
 308                 frame = ro.second;
 309         }
 310
 311         Time const relative_time = _film->audio_frames_to_time (frame);
 312
 313         if (content->trimmed (relative_time)) {
 314                 return;
 315         }
 316
 317         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time;
 318
 319         /* Remap channels */
 320         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 321         dcp_mapped->make_silent ();
 322         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 323         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 324                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
 325                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 326                 }
 327         }
 328
 329         audio = dcp_mapped;
 330
 331         /* We must cut off anything that comes before the start of all time */
 332         if (time < 0) {
 333                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 334                 if (frames >= audio->frames ()) {
 335                         return;
 336                 }
 337
 338                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 339                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 340
 341                 audio = trimmed;
 342                 time = 0;
 343         }
 344
 345         _audio_merger.push (audio, time);
 346         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 347 }
 348
 349 void
 350 Player::flush ()
 351 {
 352         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 353         if (tb.audio) {
 354                 Audio (tb.audio, tb.time);
 355                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 356         }
 357
 358         while (_video_position < _audio_position) {
 359                 emit_black ();
 360         }
 361
 362         while (_audio_position < _video_position) {
 363                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 364         }
 365
 366 }
 367
 368 /** Seek so that the next pass() will yield (approximately) the requested frame.
 369  *  Pass accurate = true to try harder to get close to the request.
 370  *  @return true on error
 371  */
 372 void
 373 Player::seek (Time t, bool accurate)
 374 {
 375         if (!_have_valid_pieces) {
 376                 setup_pieces ();
 377                 _have_valid_pieces = true;
 378         }
 379
 380         if (_pieces.empty ()) {
 381                 return;
 382         }
 383
 384         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 385                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 386                 if (!vc) {
 387                         continue;
 388                 }
 389
 390                 Time s = t - vc->position ();
 391                 s = max (static_cast<Time> (0), s);
 392                 s = min (vc->length_after_trim(), s);
 393
 394                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 395
 396                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
 397                 /* Here we are converting from time (in the DCP) to a frame number in the content.
 398                    Hence we need to use the DCP's frame rate and the double/skip correction, not
 399                    the source's rate.
 400                 */
 401                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
 402                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 403         }
 404
 405         _video_position = _audio_position = t;
 406
 407         /* XXX: don't seek audio because we don't need to... */
 408 }
 409
 410 void
 411 Player::setup_pieces ()
 412 {
 413         list<shared_ptr<Piece> > old_pieces = _pieces;
 414
 415         _pieces.clear ();
 416
 417         ContentList content = _playlist->content ();
 418         sort (content.begin(), content.end(), ContentSorter ());
 419
 420         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 421
 422                 shared_ptr<Piece> piece (new Piece (*i));
 423
 424                 /* XXX: into content? */
 425
 426                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 427                 if (fc) {
 428                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 429
 430                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 431                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 432                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
 433
 434                         piece->decoder = fd;
 435                 }
 436
 437                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
 438                 if (ic) {
 439                         shared_ptr<StillImageDecoder> id;
 440
 441                         /* See if we can re-use an old StillImageDecoder */
 442                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 443                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
 444                                 if (imd && imd->content() == ic) {
 445                                         id = imd;
 446                                 }
 447                         }
 448
 449                         if (!id) {
 450                                 id.reset (new StillImageDecoder (_film, ic));
 451                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 452                         }
 453
 454                         piece->decoder = id;
 455                 }
 456
 457                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
 458                 if (mc) {
 459                         shared_ptr<MovingImageDecoder> md;
 460
 461                         if (!md) {
 462                                 md.reset (new MovingImageDecoder (_film, mc));
 463                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 464                         }
 465
 466                         piece->decoder = md;
 467                 }
 468
 469                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 470                 if (sc) {
 471                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 472                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 473
 474                         piece->decoder = sd;
 475                 }
 476
 477                 _pieces.push_back (piece);
 478         }
 479
 480 #ifdef DEBUG_PLAYER
 481         cout << "=== Player setup:\n";
 482         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 483                 cout << *(i->get()) << "\n";
 484         }
 485 #endif
 486 }
 487
 488 void
 489 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 490 {
 491         shared_ptr<Content> c = w.lock ();
 492         if (!c) {
 493                 return;
 494         }
 495
 496         if (
 497                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 498                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 499                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO
 500                 ) {
 501
 502                 _have_valid_pieces = false;
 503                 Changed (frequent);
 504
 505         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
 506                 update_subtitle ();
 507                 Changed (frequent);
 508         } else if (property == VideoContentProperty::VIDEO_FRAME_TYPE) {
 509                 Changed (frequent);
 510         }
 511 }
 512
 513 void
 514 Player::playlist_changed ()
 515 {
 516         _have_valid_pieces = false;
 517         Changed (false);
 518 }
 519
 520 void
 521 Player::set_video_container_size (libdcp::Size s)
 522 {
 523         _video_container_size = s;
 524         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 525         _black_frame->make_black ();
 526 }
 527
 528 shared_ptr<Resampler>
 529 Player::resampler (shared_ptr<AudioContent> c, bool create)
 530 {
 531         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 532         if (i != _resamplers.end ()) {
 533                 return i->second;
 534         }
 535
 536         if (!create) {
 537                 return shared_ptr<Resampler> ();
 538         }
 539
 540         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 541         _resamplers[c] = r;
 542         return r;
 543 }
 544
 545 void
 546 Player::emit_black ()
 547 {
 548 #ifdef DCPOMATIC_DEBUG
 549         _last_video.reset ();
 550 #endif
 551
 552         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
 553         _video_position += _film->video_frames_to_time (1);
 554         _last_emit_was_black = true;
 555 }
 556
 557 void
 558 Player::emit_silence (OutputAudioFrame most)
 559 {
 560         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 561         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 562         silence->make_silent ();
 563         Audio (silence, _audio_position);
 564         _audio_position += _film->audio_frames_to_time (N);
 565 }
 566
 567 void
 568 Player::film_changed (Film::Property p)
 569 {
 570         /* Here we should notice Film properties that affect our output, and
 571            alert listeners that our output now would be different to how it was
 572            last time we were run.
 573         */
 574
 575         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 576                 Changed (false);
 577         }
 578 }
 579
 580 void
 581 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 582 {
 583         _in_subtitle.piece = weak_piece;
 584         _in_subtitle.image = image;
 585         _in_subtitle.rect = rect;
 586         _in_subtitle.from = from;
 587         _in_subtitle.to = to;
 588
 589         update_subtitle ();
 590 }
 591
 592 void
 593 Player::update_subtitle ()
 594 {
 595         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
 596         if (!piece) {
 597                 return;
 598         }
 599
 600         if (!_in_subtitle.image) {
 601                 _out_subtitle.image.reset ();
 602                 return;
 603         }
 604
 605         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 606         assert (sc);
 607
 608         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
 609         libdcp::Size scaled_size;
 610
 611         in_rect.y += sc->subtitle_offset ();
 612
 613         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 614         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
 615         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
 616
 617         /* Then we need a corrective translation, consisting of two parts:
 618          *
 619          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 620          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 621          *
 622          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 623          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 624          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 625          *
 626          * Combining these two translations gives these expressions.
 627          */
 628
 629         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
 630         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
 631
 632         _out_subtitle.image = _in_subtitle.image->scale (libdcp::Size (scaled_size.width, scaled_size.height), Scaler::from_id ("bicubic"), true);
 633         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
 634         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
 635 }