src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "still_image_decoder.h"
  26 #include "still_image_content.h"
  27 #include "moving_image_decoder.h"
  28 #include "moving_image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "playlist.h"
  33 #include "job.h"
  34 #include "image.h"
  35 #include "ratio.h"
  36 #include "resampler.h"
  37 #include "log.h"
  38 #include "scaler.h"
  39
  40 using std::list;
  41 using std::cout;
  42 using std::min;
  43 using std::max;
  44 using std::vector;
  45 using std::pair;
  46 using std::map;
  47 using boost::shared_ptr;
  48 using boost::weak_ptr;
  49 using boost::dynamic_pointer_cast;
  50
  51 //#define DEBUG_PLAYER 1
  52
  53 class Piece
  54 {
  55 public:
  56         Piece (shared_ptr<Content> c)
  57                 : content (c)
  58                 , video_position (c->position ())
  59                 , audio_position (c->position ())
  60         {}
  61
  62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  63                 : content (c)
  64                 , decoder (d)
  65                 , video_position (c->position ())
  66                 , audio_position (c->position ())
  67         {}
  68
  69         shared_ptr<Content> content;
  70         shared_ptr<Decoder> decoder;
  71         Time video_position;
  72         Time audio_position;
  73 };
  74
  75 #ifdef DEBUG_PLAYER
  76 std::ostream& operator<<(std::ostream& s, Piece const & p)
  77 {
  78         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
  79                 s << "\tffmpeg     ";
  80         } else if (dynamic_pointer_cast<StillImageContent> (p.content)) {
  81                 s << "\tstill image";
  82         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
  83                 s << "\tsndfile    ";
  84         }
  85
  86         s << " at " << p.content->position() << " until " << p.content->end();
  87
  88         return s;
  89 }
  90 #endif
  91
  92 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  93         : _film (f)
  94         , _playlist (p)
  95         , _video (true)
  96         , _audio (true)
  97         , _have_valid_pieces (false)
  98         , _video_position (0)
  99         , _audio_position (0)
 100         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
 101         , _last_emit_was_black (false)
 102 {
 103         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 104         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
 105         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
 106         set_video_container_size (_film->container()->size (_film->full_frame ()));
 107 }
 108
 109 void
 110 Player::disable_video ()
 111 {
 112         _video = false;
 113 }
 114
 115 void
 116 Player::disable_audio ()
 117 {
 118         _audio = false;
 119 }
 120
 121 bool
 122 Player::pass ()
 123 {
 124         if (!_have_valid_pieces) {
 125                 setup_pieces ();
 126                 _have_valid_pieces = true;
 127         }
 128
 129 #ifdef DEBUG_PLAYER
 130         cout << "= PASS\n";
 131 #endif
 132
 133         Time earliest_t = TIME_MAX;
 134         shared_ptr<Piece> earliest;
 135         enum {
 136                 VIDEO,
 137                 AUDIO
 138         } type = VIDEO;
 139
 140         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 141                 if ((*i)->decoder->done ()) {
 142                         continue;
 143                 }
 144
 145                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 146                         if ((*i)->video_position < earliest_t) {
 147                                 earliest_t = (*i)->video_position;
 148                                 earliest = *i;
 149                                 type = VIDEO;
 150                         }
 151                 }
 152
 153                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 154                         if ((*i)->audio_position < earliest_t) {
 155                                 earliest_t = (*i)->audio_position;
 156                                 earliest = *i;
 157                                 type = AUDIO;
 158                         }
 159                 }
 160         }
 161
 162         if (!earliest) {
 163 #ifdef DEBUG_PLAYER
 164                 cout << "no earliest piece.\n";
 165 #endif
 166
 167                 flush ();
 168                 return true;
 169         }
 170
 171         switch (type) {
 172         case VIDEO:
 173                 if (earliest_t > _video_position) {
 174 #ifdef DEBUG_PLAYER
 175                         cout << "no video here; emitting black frame (earliest=" << earliest_t << ", video_position=" << _video_position << ").\n";
 176 #endif
 177                         emit_black ();
 178                 } else {
 179 #ifdef DEBUG_PLAYER
 180                         cout << "Pass " << *earliest << "\n";
 181 #endif
 182                         earliest->decoder->pass ();
 183                 }
 184                 break;
 185
 186         case AUDIO:
 187                 if (earliest_t > _audio_position) {
 188 #ifdef DEBUG_PLAYER
 189                         cout << "no audio here; emitting silence.\n";
 190 #endif
 191                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 192                 } else {
 193 #ifdef DEBUG_PLAYER
 194                         cout << "Pass " << *earliest << "\n";
 195 #endif
 196                         earliest->decoder->pass ();
 197
 198                         if (earliest->decoder->done()) {
 199                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 200                                 assert (ac);
 201                                 shared_ptr<Resampler> re = resampler (ac, false);
 202                                 if (re) {
 203                                         shared_ptr<const AudioBuffers> b = re->flush ();
 204                                         if (b->frames ()) {
 205                                                 process_audio (earliest, b, ac->audio_length ());
 206                                         }
 207                                 }
 208                         }
 209
 210
 211                 }
 212
 213                 Time done_up_to = TIME_MAX;
 214                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 215                         if (dynamic_pointer_cast<AudioContent> ((*i)->content)) {
 216                                 done_up_to = min (done_up_to, (*i)->audio_position);
 217                         }
 218                 }
 219
 220                 TimedAudioBuffers<Time> tb = _audio_merger.pull (done_up_to);
 221                 Audio (tb.audio, tb.time);
 222                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 223                 break;
 224         }
 225
 226
 227
 228 #ifdef DEBUG_PLAYER
 229         cout << "\tpost pass " << _video_position << " " << _audio_position << "\n";
 230 #endif
 231
 232         return false;
 233 }
 234
 235 void
 236 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
 237 {
 238         shared_ptr<Piece> piece = weak_piece.lock ();
 239         if (!piece) {
 240                 return;
 241         }
 242
 243         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 244         assert (content);
 245
 246         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 247         if (frc.skip && (frame % 2) == 1) {
 248                 return;
 249         }
 250
 251         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 252         if (content->trimmed (relative_time)) {
 253                 return;
 254         }
 255
 256         shared_ptr<Image> work_image = image->crop (content->crop(), true);
 257
 258         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
 259
 260         work_image = work_image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
 261
 262         Time time = content->position() + relative_time - content->trim_start ();
 263
 264         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
 265                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
 266         }
 267
 268         if (image_size != _video_container_size) {
 269                 assert (image_size.width <= _video_container_size.width);
 270                 assert (image_size.height <= _video_container_size.height);
 271                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 272                 im->make_black ();
 273                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 274                 work_image = im;
 275         }
 276
 277 #ifdef DCPOMATIC_DEBUG
 278         _last_video = piece->content;
 279 #endif
 280
 281         Video (work_image, eyes, same, time);
 282         time += TIME_HZ / _film->video_frame_rate();
 283
 284         if (frc.repeat) {
 285                 Video (work_image, eyes, true, time);
 286                 time += TIME_HZ / _film->video_frame_rate();
 287         }
 288
 289         _last_emit_was_black = false;
 290
 291         _video_position = piece->video_position = time;
 292 }
 293
 294 void
 295 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 296 {
 297         shared_ptr<Piece> piece = weak_piece.lock ();
 298         if (!piece) {
 299                 return;
 300         }
 301
 302         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 303         assert (content);
 304
 305         Time const relative_time = _film->audio_frames_to_time (frame)
 306                 + (content->audio_delay() * TIME_HZ / 1000);
 307
 308         if (content->trimmed (relative_time)) {
 309                 return;
 310         }
 311
 312         Time time = content->position() + relative_time;
 313
 314         /* Resample */
 315         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 316                 shared_ptr<Resampler> r = resampler (content, true);
 317                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 318                 audio = ro.first;
 319                 frame = ro.second;
 320         }
 321
 322         /* Remap channels */
 323         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 324         dcp_mapped->make_silent ();
 325         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 326         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 327                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
 328                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 329                 }
 330         }
 331
 332         audio = dcp_mapped;
 333
 334         /* We must cut off anything that comes before the start of all time */
 335         if (time < 0) {
 336                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 337                 if (frames >= audio->frames ()) {
 338                         return;
 339                 }
 340
 341                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 342                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 343
 344                 audio = trimmed;
 345                 time = 0;
 346         }
 347
 348         _audio_merger.push (audio, time);
 349         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 350 }
 351
 352 void
 353 Player::flush ()
 354 {
 355         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 356         if (tb.audio) {
 357                 Audio (tb.audio, tb.time);
 358                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 359         }
 360
 361         while (_video_position < _audio_position) {
 362                 emit_black ();
 363         }
 364
 365         while (_audio_position < _video_position) {
 366                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 367         }
 368
 369 }
 370
 371 /** Seek so that the next pass() will yield (approximately) the requested frame.
 372  *  Pass accurate = true to try harder to get close to the request.
 373  *  @return true on error
 374  */
 375 void
 376 Player::seek (Time t, bool accurate)
 377 {
 378         if (!_have_valid_pieces) {
 379                 setup_pieces ();
 380                 _have_valid_pieces = true;
 381         }
 382
 383         if (_pieces.empty ()) {
 384                 return;
 385         }
 386
 387         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 388                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 389                 if (!vc) {
 390                         continue;
 391                 }
 392
 393                 Time s = t - vc->position ();
 394                 s = max (static_cast<Time> (0), s);
 395                 s = min (vc->length_after_trim(), s);
 396
 397                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 398
 399                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
 400                 /* Here we are converting from time (in the DCP) to a frame number in the content.
 401                    Hence we need to use the DCP's frame rate and the double/skip correction, not
 402                    the source's rate.
 403                 */
 404                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
 405                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 406         }
 407
 408         _video_position = _audio_position = t;
 409
 410         /* XXX: don't seek audio because we don't need to... */
 411 }
 412
 413 void
 414 Player::setup_pieces ()
 415 {
 416         list<shared_ptr<Piece> > old_pieces = _pieces;
 417
 418         _pieces.clear ();
 419
 420         ContentList content = _playlist->content ();
 421         sort (content.begin(), content.end(), ContentSorter ());
 422
 423         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 424
 425                 shared_ptr<Piece> piece (new Piece (*i));
 426
 427                 /* XXX: into content? */
 428
 429                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 430                 if (fc) {
 431                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 432
 433                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 434                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 435                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
 436
 437                         piece->decoder = fd;
 438                 }
 439
 440                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
 441                 if (ic) {
 442                         shared_ptr<StillImageDecoder> id;
 443
 444                         /* See if we can re-use an old StillImageDecoder */
 445                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 446                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
 447                                 if (imd && imd->content() == ic) {
 448                                         id = imd;
 449                                 }
 450                         }
 451
 452                         if (!id) {
 453                                 id.reset (new StillImageDecoder (_film, ic));
 454                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 455                         }
 456
 457                         piece->decoder = id;
 458                 }
 459
 460                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
 461                 if (mc) {
 462                         shared_ptr<MovingImageDecoder> md;
 463
 464                         if (!md) {
 465                                 md.reset (new MovingImageDecoder (_film, mc));
 466                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 467                         }
 468
 469                         piece->decoder = md;
 470                 }
 471
 472                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 473                 if (sc) {
 474                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 475                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 476
 477                         piece->decoder = sd;
 478                 }
 479
 480                 _pieces.push_back (piece);
 481         }
 482
 483 #ifdef DEBUG_PLAYER
 484         cout << "=== Player setup:\n";
 485         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 486                 cout << *(i->get()) << "\n";
 487         }
 488 #endif
 489 }
 490
 491 void
 492 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 493 {
 494         shared_ptr<Content> c = w.lock ();
 495         if (!c) {
 496                 return;
 497         }
 498
 499         if (
 500                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 501                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 502                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO
 503                 ) {
 504
 505                 _have_valid_pieces = false;
 506                 Changed (frequent);
 507
 508         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
 509                 update_subtitle ();
 510                 Changed (frequent);
 511         } else if (property == VideoContentProperty::VIDEO_FRAME_TYPE) {
 512                 Changed (frequent);
 513         }
 514 }
 515
 516 void
 517 Player::playlist_changed ()
 518 {
 519         _have_valid_pieces = false;
 520         Changed (false);
 521 }
 522
 523 void
 524 Player::set_video_container_size (libdcp::Size s)
 525 {
 526         _video_container_size = s;
 527         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 528         _black_frame->make_black ();
 529 }
 530
 531 shared_ptr<Resampler>
 532 Player::resampler (shared_ptr<AudioContent> c, bool create)
 533 {
 534         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 535         if (i != _resamplers.end ()) {
 536                 return i->second;
 537         }
 538
 539         if (!create) {
 540                 return shared_ptr<Resampler> ();
 541         }
 542
 543         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 544         _resamplers[c] = r;
 545         return r;
 546 }
 547
 548 void
 549 Player::emit_black ()
 550 {
 551 #ifdef DCPOMATIC_DEBUG
 552         _last_video.reset ();
 553 #endif
 554
 555         Video (_black_frame, EYES_BOTH, _last_emit_was_black, _video_position);
 556         _video_position += _film->video_frames_to_time (1);
 557         _last_emit_was_black = true;
 558 }
 559
 560 void
 561 Player::emit_silence (OutputAudioFrame most)
 562 {
 563         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 564         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 565         silence->make_silent ();
 566         Audio (silence, _audio_position);
 567         _audio_position += _film->audio_frames_to_time (N);
 568 }
 569
 570 void
 571 Player::film_changed (Film::Property p)
 572 {
 573         /* Here we should notice Film properties that affect our output, and
 574            alert listeners that our output now would be different to how it was
 575            last time we were run.
 576         */
 577
 578         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 579                 Changed (false);
 580         }
 581 }
 582
 583 void
 584 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 585 {
 586         _in_subtitle.piece = weak_piece;
 587         _in_subtitle.image = image;
 588         _in_subtitle.rect = rect;
 589         _in_subtitle.from = from;
 590         _in_subtitle.to = to;
 591
 592         update_subtitle ();
 593 }
 594
 595 void
 596 Player::update_subtitle ()
 597 {
 598         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
 599         if (!piece) {
 600                 return;
 601         }
 602
 603         if (!_in_subtitle.image) {
 604                 _out_subtitle.image.reset ();
 605                 return;
 606         }
 607
 608         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 609         assert (sc);
 610
 611         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
 612         libdcp::Size scaled_size;
 613
 614         in_rect.y += sc->subtitle_offset ();
 615
 616         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 617         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
 618         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
 619
 620         /* Then we need a corrective translation, consisting of two parts:
 621          *
 622          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 623          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 624          *
 625          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 626          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 627          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 628          *
 629          * Combining these two translations gives these expressions.
 630          */
 631
 632         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
 633         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
 634
 635         _out_subtitle.image = _in_subtitle.image->scale (libdcp::Size (scaled_size.width, scaled_size.height), Scaler::from_id ("bicubic"), true);
 636         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
 637         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
 638 }