src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "still_image_decoder.h"
  26 #include "still_image_content.h"
  27 #include "moving_image_decoder.h"
  28 #include "moving_image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "playlist.h"
  33 #include "job.h"
  34 #include "image.h"
  35 #include "ratio.h"
  36 #include "resampler.h"
  37 #include "log.h"
  38 #include "scaler.h"
  39
  40 using std::list;
  41 using std::cout;
  42 using std::min;
  43 using std::max;
  44 using std::vector;
  45 using std::pair;
  46 using std::map;
  47 using boost::shared_ptr;
  48 using boost::weak_ptr;
  49 using boost::dynamic_pointer_cast;
  50
  51 //#define DEBUG_PLAYER 1
  52
  53 class Piece
  54 {
  55 public:
  56         Piece (shared_ptr<Content> c)
  57                 : content (c)
  58                 , video_position (c->position ())
  59                 , audio_position (c->position ())
  60         {}
  61
  62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  63                 : content (c)
  64                 , decoder (d)
  65                 , video_position (c->position ())
  66                 , audio_position (c->position ())
  67         {}
  68
  69         shared_ptr<Content> content;
  70         shared_ptr<Decoder> decoder;
  71         Time video_position;
  72         Time audio_position;
  73 };
  74
  75 #ifdef DEBUG_PLAYER
  76 std::ostream& operator<<(std::ostream& s, Piece const & p)
  77 {
  78         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
  79                 s << "\tffmpeg     ";
  80         } else if (dynamic_pointer_cast<StillImageContent> (p.content)) {
  81                 s << "\tstill image";
  82         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
  83                 s << "\tsndfile    ";
  84         }
  85
  86         s << " at " << p.content->position() << " until " << p.content->end();
  87
  88         return s;
  89 }
  90 #endif
  91
  92 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  93         : _film (f)
  94         , _playlist (p)
  95         , _video (true)
  96         , _audio (true)
  97         , _have_valid_pieces (false)
  98         , _video_position (0)
  99         , _audio_position (0)
 100         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
 101         , _last_emit_was_black (false)
 102 {
 103         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 104         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
 105         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
 106         set_video_container_size (_film->container()->size (_film->full_frame ()));
 107 }
 108
 109 void
 110 Player::disable_video ()
 111 {
 112         _video = false;
 113 }
 114
 115 void
 116 Player::disable_audio ()
 117 {
 118         _audio = false;
 119 }
 120
 121 bool
 122 Player::pass ()
 123 {
 124         if (!_have_valid_pieces) {
 125                 setup_pieces ();
 126                 _have_valid_pieces = true;
 127         }
 128
 129 #ifdef DEBUG_PLAYER
 130         cout << "= PASS\n";
 131 #endif
 132
 133         Time earliest_t = TIME_MAX;
 134         shared_ptr<Piece> earliest;
 135         enum {
 136                 VIDEO,
 137                 AUDIO
 138         } type = VIDEO;
 139
 140         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 141                 if ((*i)->decoder->done ()) {
 142                         continue;
 143                 }
 144
 145                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 146                         if ((*i)->video_position < earliest_t) {
 147                                 earliest_t = (*i)->video_position;
 148                                 earliest = *i;
 149                                 type = VIDEO;
 150                         }
 151                 }
 152
 153                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 154                         if ((*i)->audio_position < earliest_t) {
 155                                 earliest_t = (*i)->audio_position;
 156                                 earliest = *i;
 157                                 type = AUDIO;
 158                         }
 159                 }
 160         }
 161
 162         if (!earliest) {
 163 #ifdef DEBUG_PLAYER
 164                 cout << "no earliest piece.\n";
 165 #endif
 166
 167                 flush ();
 168                 return true;
 169         }
 170
 171         switch (type) {
 172         case VIDEO:
 173                 if (earliest_t > _video_position) {
 174 #ifdef DEBUG_PLAYER
 175                         cout << "no video here; emitting black frame (earliest=" << earliest_t << ", video_position=" << _video_position << ").\n";
 176 #endif
 177                         emit_black ();
 178                 } else {
 179 #ifdef DEBUG_PLAYER
 180                         cout << "Pass video " << *earliest << "\n";
 181 #endif
 182                         earliest->decoder->pass ();
 183                 }
 184                 break;
 185
 186         case AUDIO:
 187                 if (earliest_t > _audio_position) {
 188 #ifdef DEBUG_PLAYER
 189                         cout << "no audio here (none until " << earliest_t << "); emitting silence.\n";
 190 #endif
 191                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 192                 } else {
 193 #ifdef DEBUG_PLAYER
 194                         cout << "Pass audio " << *earliest << "\n";
 195 #endif
 196                         earliest->decoder->pass ();
 197
 198                         if (earliest->decoder->done()) {
 199                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 200                                 assert (ac);
 201                                 shared_ptr<Resampler> re = resampler (ac, false);
 202                                 if (re) {
 203                                         shared_ptr<const AudioBuffers> b = re->flush ();
 204                                         if (b->frames ()) {
 205                                                 process_audio (earliest, b, ac->audio_length ());
 206                                         }
 207                                 }
 208                         }
 209                 }
 210                 break;
 211         }
 212
 213         Time audio_done_up_to = TIME_MAX;
 214         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 215                 if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 216                         audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
 217                 }
 218         }
 219
 220         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
 221         Audio (tb.audio, tb.time);
 222         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 223
 224 #ifdef DEBUG_PLAYER
 225         cout << "\tpost pass _video_position=" << _video_position << " _audio_position=" << _audio_position << "\n";
 226 #endif
 227
 228         return false;
 229 }
 230
 231 void
 232 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
 233 {
 234         shared_ptr<Piece> piece = weak_piece.lock ();
 235         if (!piece) {
 236                 return;
 237         }
 238
 239         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 240         assert (content);
 241
 242         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 243         if (frc.skip && (frame % 2) == 1) {
 244                 return;
 245         }
 246
 247         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 248         if (content->trimmed (relative_time)) {
 249                 return;
 250         }
 251
 252         shared_ptr<Image> work_image = image->crop (content->crop(), true);
 253
 254         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
 255
 256         work_image = work_image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
 257
 258         Time time = content->position() + relative_time - content->trim_start ();
 259
 260         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
 261                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
 262         }
 263
 264         if (image_size != _video_container_size) {
 265                 assert (image_size.width <= _video_container_size.width);
 266                 assert (image_size.height <= _video_container_size.height);
 267                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 268                 im->make_black ();
 269                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 270                 work_image = im;
 271         }
 272
 273 #ifdef DCPOMATIC_DEBUG
 274         _last_video = piece->content;
 275 #endif
 276
 277         Video (work_image, eyes, same, time);
 278         time += TIME_HZ / _film->video_frame_rate();
 279
 280         if (frc.repeat) {
 281                 Video (work_image, eyes, true, time);
 282                 time += TIME_HZ / _film->video_frame_rate();
 283         }
 284
 285         _last_emit_was_black = false;
 286
 287         _video_position = piece->video_position = time;
 288 }
 289
 290 void
 291 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 292 {
 293         shared_ptr<Piece> piece = weak_piece.lock ();
 294         if (!piece) {
 295                 return;
 296         }
 297
 298         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 299         assert (content);
 300
 301         /* Resample */
 302         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 303                 shared_ptr<Resampler> r = resampler (content, true);
 304                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 305                 audio = ro.first;
 306                 frame = ro.second;
 307         }
 308
 309         Time const relative_time = _film->audio_frames_to_time (frame);
 310
 311         if (content->trimmed (relative_time)) {
 312                 return;
 313         }
 314
 315         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time;
 316
 317         /* Remap channels */
 318         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 319         dcp_mapped->make_silent ();
 320         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 321         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 322                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
 323                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 324                 }
 325         }
 326
 327         audio = dcp_mapped;
 328
 329         /* We must cut off anything that comes before the start of all time */
 330         if (time < 0) {
 331                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 332                 if (frames >= audio->frames ()) {
 333                         return;
 334                 }
 335
 336                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 337                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 338
 339                 audio = trimmed;
 340                 time = 0;
 341         }
 342
 343         _audio_merger.push (audio, time);
 344         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 345 }
 346
 347 void
 348 Player::flush ()
 349 {
 350         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 351         if (tb.audio) {
 352                 Audio (tb.audio, tb.time);
 353                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 354         }
 355
 356         while (_video_position < _audio_position) {
 357                 emit_black ();
 358         }
 359
 360         while (_audio_position < _video_position) {
 361                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 362         }
 363
 364 }
 365
 366 /** Seek so that the next pass() will yield (approximately) the requested frame.
 367  *  Pass accurate = true to try harder to get close to the request.
 368  *  @return true on error
 369  */
 370 void
 371 Player::seek (Time t, bool accurate)
 372 {
 373         if (!_have_valid_pieces) {
 374                 setup_pieces ();
 375                 _have_valid_pieces = true;
 376         }
 377
 378         if (_pieces.empty ()) {
 379                 return;
 380         }
 381
 382         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 383                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 384                 if (!vc) {
 385                         continue;
 386                 }
 387
 388                 Time s = t - vc->position ();
 389                 s = max (static_cast<Time> (0), s);
 390                 s = min (vc->length_after_trim(), s);
 391
 392                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 393
 394                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
 395                 /* Here we are converting from time (in the DCP) to a frame number in the content.
 396                    Hence we need to use the DCP's frame rate and the double/skip correction, not
 397                    the source's rate.
 398                 */
 399                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
 400                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 401         }
 402
 403         _video_position = _audio_position = t;
 404
 405         /* XXX: don't seek audio because we don't need to... */
 406 }
 407
 408 void
 409 Player::setup_pieces ()
 410 {
 411         list<shared_ptr<Piece> > old_pieces = _pieces;
 412
 413         _pieces.clear ();
 414
 415         ContentList content = _playlist->content ();
 416         sort (content.begin(), content.end(), ContentSorter ());
 417
 418         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 419
 420                 shared_ptr<Piece> piece (new Piece (*i));
 421
 422                 /* XXX: into content? */
 423
 424                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 425                 if (fc) {
 426                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 427
 428                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 429                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 430                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
 431
 432                         piece->decoder = fd;
 433                 }
 434
 435                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
 436                 if (ic) {
 437                         shared_ptr<StillImageDecoder> id;
 438
 439                         /* See if we can re-use an old StillImageDecoder */
 440                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 441                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
 442                                 if (imd && imd->content() == ic) {
 443                                         id = imd;
 444                                 }
 445                         }
 446
 447                         if (!id) {
 448                                 id.reset (new StillImageDecoder (_film, ic));
 449                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 450                         }
 451
 452                         piece->decoder = id;
 453                 }
 454
 455                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
 456                 if (mc) {
 457                         shared_ptr<MovingImageDecoder> md;
 458
 459                         if (!md) {
 460                                 md.reset (new MovingImageDecoder (_film, mc));
 461                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 462                         }
 463
 464                         piece->decoder = md;
 465                 }
 466
 467                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 468                 if (sc) {
 469                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 470                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 471
 472                         piece->decoder = sd;
 473                 }
 474
 475                 _pieces.push_back (piece);
 476         }
 477
 478 #ifdef DEBUG_PLAYER
 479         cout << "=== Player setup:\n";
 480         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 481                 cout << *(i->get()) << "\n";
 482         }
 483 #endif
 484 }
 485
 486 void
 487 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 488 {
 489         shared_ptr<Content> c = w.lock ();
 490         if (!c) {
 491                 return;
 492         }
 493
 494         if (
 495                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 496                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 497                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO
 498                 ) {
 499
 500                 _have_valid_pieces = false;
 501                 Changed (frequent);
 502
 503         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
 504                 update_subtitle ();
 505                 Changed (frequent);
 506         } else if (property == VideoContentProperty::VIDEO_FRAME_TYPE) {
 507                 Changed (frequent);
 508         }
 509 }
 510
 511 void
 512 Player::playlist_changed ()
 513 {
 514         _have_valid_pieces = false;
 515         Changed (false);
 516 }
 517
 518 void
 519 Player::set_video_container_size (libdcp::Size s)
 520 {
 521         _video_container_size = s;
 522         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 523         _black_frame->make_black ();
 524 }
 525
 526 shared_ptr<Resampler>
 527 Player::resampler (shared_ptr<AudioContent> c, bool create)
 528 {
 529         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 530         if (i != _resamplers.end ()) {
 531                 return i->second;
 532         }
 533
 534         if (!create) {
 535                 return shared_ptr<Resampler> ();
 536         }
 537
 538         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 539         _resamplers[c] = r;
 540         return r;
 541 }
 542
 543 void
 544 Player::emit_black ()
 545 {
 546 #ifdef DCPOMATIC_DEBUG
 547         _last_video.reset ();
 548 #endif
 549
 550         Video (_black_frame, EYES_BOTH, _last_emit_was_black, _video_position);
 551         _video_position += _film->video_frames_to_time (1);
 552         _last_emit_was_black = true;
 553 }
 554
 555 void
 556 Player::emit_silence (OutputAudioFrame most)
 557 {
 558         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 559         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 560         silence->make_silent ();
 561         Audio (silence, _audio_position);
 562         _audio_position += _film->audio_frames_to_time (N);
 563 }
 564
 565 void
 566 Player::film_changed (Film::Property p)
 567 {
 568         /* Here we should notice Film properties that affect our output, and
 569            alert listeners that our output now would be different to how it was
 570            last time we were run.
 571         */
 572
 573         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 574                 Changed (false);
 575         }
 576 }
 577
 578 void
 579 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 580 {
 581         _in_subtitle.piece = weak_piece;
 582         _in_subtitle.image = image;
 583         _in_subtitle.rect = rect;
 584         _in_subtitle.from = from;
 585         _in_subtitle.to = to;
 586
 587         update_subtitle ();
 588 }
 589
 590 void
 591 Player::update_subtitle ()
 592 {
 593         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
 594         if (!piece) {
 595                 return;
 596         }
 597
 598         if (!_in_subtitle.image) {
 599                 _out_subtitle.image.reset ();
 600                 return;
 601         }
 602
 603         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 604         assert (sc);
 605
 606         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
 607         libdcp::Size scaled_size;
 608
 609         in_rect.y += sc->subtitle_offset ();
 610
 611         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 612         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
 613         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
 614
 615         /* Then we need a corrective translation, consisting of two parts:
 616          *
 617          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 618          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 619          *
 620          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 621          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 622          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 623          *
 624          * Combining these two translations gives these expressions.
 625          */
 626
 627         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
 628         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
 629
 630         _out_subtitle.image = _in_subtitle.image->scale (libdcp::Size (scaled_size.width, scaled_size.height), Scaler::from_id ("bicubic"), true);
 631         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
 632         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
 633 }