src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "still_image_decoder.h"
  26 #include "still_image_content.h"
  27 #include "moving_image_decoder.h"
  28 #include "moving_image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "playlist.h"
  33 #include "job.h"
  34 #include "image.h"
  35 #include "ratio.h"
  36 #include "resampler.h"
  37 #include "log.h"
  38 #include "scaler.h"
  39
  40 using std::list;
  41 using std::cout;
  42 using std::min;
  43 using std::max;
  44 using std::vector;
  45 using std::pair;
  46 using std::map;
  47 using boost::shared_ptr;
  48 using boost::weak_ptr;
  49 using boost::dynamic_pointer_cast;
  50
  51 #define DEBUG_PLAYER 1
  52
  53 class Piece
  54 {
  55 public:
  56         Piece (shared_ptr<Content> c)
  57                 : content (c)
  58                 , video_position (c->start ())
  59                 , audio_position (c->start ())
  60         {}
  61
  62         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  63                 : content (c)
  64                 , decoder (d)
  65                 , video_position (c->start ())
  66                 , audio_position (c->start ())
  67         {}
  68
  69         shared_ptr<Content> content;
  70         shared_ptr<Decoder> decoder;
  71         Time video_position;
  72         Time audio_position;
  73 };
  74
  75 #ifdef DEBUG_PLAYER
  76 std::ostream& operator<<(std::ostream& s, Piece const & p)
  77 {
  78         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
  79                 s << "\tffmpeg     ";
  80         } else if (dynamic_pointer_cast<StillImageContent> (p.content)) {
  81                 s << "\tstill image";
  82         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
  83                 s << "\tsndfile    ";
  84         }
  85
  86         s << " at " << p.content->start() << " until " << p.content->end();
  87
  88         return s;
  89 }
  90 #endif
  91
  92 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  93         : _film (f)
  94         , _playlist (p)
  95         , _video (true)
  96         , _audio (true)
  97         , _have_valid_pieces (false)
  98         , _video_position (0)
  99         , _audio_position (0)
 100         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
 101 {
 102         _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 103         _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
 104         _film->Changed.connect (bind (&Player::film_changed, this, _1));
 105         set_video_container_size (_film->container()->size (_film->full_frame ()));
 106 }
 107
 108 void
 109 Player::disable_video ()
 110 {
 111         _video = false;
 112 }
 113
 114 void
 115 Player::disable_audio ()
 116 {
 117         _audio = false;
 118 }
 119
 120 bool
 121 Player::pass ()
 122 {
 123         if (!_have_valid_pieces) {
 124                 setup_pieces ();
 125                 _have_valid_pieces = true;
 126         }
 127
 128 #ifdef DEBUG_PLAYER
 129         cout << "= PASS\n";
 130 #endif
 131
 132         Time earliest_t = TIME_MAX;
 133         shared_ptr<Piece> earliest;
 134         enum {
 135                 VIDEO,
 136                 AUDIO
 137         } type = VIDEO;
 138
 139         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 140                 if ((*i)->decoder->done ()) {
 141                         continue;
 142                 }
 143
 144                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 145                         if ((*i)->video_position < earliest_t) {
 146                                 earliest_t = (*i)->video_position;
 147                                 earliest = *i;
 148                                 type = VIDEO;
 149                         }
 150                 }
 151
 152                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 153                         if ((*i)->audio_position < earliest_t) {
 154                                 earliest_t = (*i)->audio_position;
 155                                 earliest = *i;
 156                                 type = AUDIO;
 157                         }
 158                 }
 159         }
 160
 161         if (!earliest) {
 162 #ifdef DEBUG_PLAYER
 163                 cout << "no earliest piece.\n";
 164 #endif
 165
 166                 flush ();
 167                 return true;
 168         }
 169
 170         switch (type) {
 171         case VIDEO:
 172                 if (earliest_t > _video_position) {
 173 #ifdef DEBUG_PLAYER
 174                         cout << "no video here; emitting black frame (earliest=" << earliest_t << ", video_position=" << _video_position << ").\n";
 175 #endif
 176                         emit_black ();
 177                 } else {
 178 #ifdef DEBUG_PLAYER
 179                         cout << "Pass " << *earliest << "\n";
 180 #endif
 181                         earliest->decoder->pass ();
 182                 }
 183                 break;
 184
 185         case AUDIO:
 186                 if (earliest_t > _audio_position) {
 187 #ifdef DEBUG_PLAYER
 188                         cout << "no audio here; emitting silence.\n";
 189 #endif
 190                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 191                 } else {
 192 #ifdef DEBUG_PLAYER
 193                         cout << "Pass " << *earliest << "\n";
 194 #endif
 195                         earliest->decoder->pass ();
 196
 197                         if (earliest->decoder->done()) {
 198                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 199                                 assert (ac);
 200                                 shared_ptr<Resampler> re = resampler (ac, false);
 201                                 if (re) {
 202                                         shared_ptr<const AudioBuffers> b = re->flush ();
 203                                         if (b->frames ()) {
 204                                                 process_audio (earliest, b, ac->audio_length ());
 205                                         }
 206                                 }
 207                         }
 208
 209                 }
 210                 break;
 211         }
 212
 213 #ifdef DEBUG_PLAYER
 214         cout << "\tpost pass " << _video_position << " " << _audio_position << "\n";
 215 #endif
 216
 217         return false;
 218 }
 219
 220 void
 221 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
 222 {
 223         shared_ptr<Piece> piece = weak_piece.lock ();
 224         if (!piece) {
 225                 return;
 226         }
 227
 228         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 229         assert (content);
 230
 231         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 232         if (frc.skip && (frame % 2) == 1) {
 233                 return;
 234         }
 235
 236         shared_ptr<Image> work_image = image->crop (content->crop(), true);
 237
 238         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
 239
 240         work_image = work_image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
 241
 242         Time time = content->start() + (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 243
 244         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
 245                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
 246         }
 247
 248         if (image_size != _video_container_size) {
 249                 assert (image_size.width <= _video_container_size.width);
 250                 assert (image_size.height <= _video_container_size.height);
 251                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 252                 im->make_black ();
 253                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 254                 work_image = im;
 255         }
 256
 257 #ifdef DCPOMATIC_DEBUG
 258         _last_video = piece->content;
 259 #endif
 260
 261         Video (work_image, eyes, same, time);
 262         time += TIME_HZ / _film->video_frame_rate();
 263
 264         if (frc.repeat) {
 265                 Video (work_image, eyes, true, time);
 266                 time += TIME_HZ / _film->video_frame_rate();
 267         }
 268
 269         _video_position = piece->video_position = time;
 270 }
 271
 272 void
 273 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 274 {
 275         shared_ptr<Piece> piece = weak_piece.lock ();
 276         if (!piece) {
 277                 return;
 278         }
 279
 280         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 281         assert (content);
 282
 283         /* Resample */
 284         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 285                 shared_ptr<Resampler> r = resampler (content, true);
 286                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 287                 audio = ro.first;
 288                 frame = ro.second;
 289         }
 290
 291         /* Remap channels */
 292         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 293         dcp_mapped->make_silent ();
 294         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 295         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 296                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
 297                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 298                 }
 299         }
 300
 301         audio = dcp_mapped;
 302
 303         Time time = content->start()
 304                 + _film->audio_frames_to_time (frame)
 305                 + (content->audio_delay() * TIME_HZ / 1000);
 306
 307         /* We must cut off anything that comes before the start of all time */
 308         if (time < 0) {
 309                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 310                 if (frames >= audio->frames ()) {
 311                         return;
 312                 }
 313
 314                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 315                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 316
 317                 audio = trimmed;
 318                 time = 0;
 319         }
 320
 321         cout << "push " << audio->frames() << " @ " << time << " from " << content->path() << "\n";
 322         TimedAudioBuffers<Time> tb = _audio_merger.push (audio, time);
 323         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 324
 325         if (tb.audio) {
 326                 Audio (tb.audio, tb.time);
 327                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 328                 cout << "output " << tb.audio->frames() << " @ " << tb.time << "\n";
 329         }
 330 }
 331
 332 void
 333 Player::flush ()
 334 {
 335         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 336         if (tb.audio) {
 337                 Audio (tb.audio, tb.time);
 338                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 339         }
 340
 341         while (_video_position < _audio_position) {
 342                 emit_black ();
 343         }
 344
 345         while (_audio_position < _video_position) {
 346                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 347         }
 348
 349 }
 350
 351 /** Seek so that the next pass() will yield (approximately) the requested frame.
 352  *  Pass accurate = true to try harder to get close to the request.
 353  *  @return true on error
 354  */
 355 void
 356 Player::seek (Time t, bool accurate)
 357 {
 358         if (!_have_valid_pieces) {
 359                 setup_pieces ();
 360                 _have_valid_pieces = true;
 361         }
 362
 363         if (_pieces.empty ()) {
 364                 return;
 365         }
 366
 367         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 368                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 369                 if (!vc) {
 370                         continue;
 371                 }
 372
 373                 Time s = t - vc->start ();
 374                 s = max (static_cast<Time> (0), s);
 375                 s = min (vc->length(), s);
 376
 377                 (*i)->video_position = (*i)->audio_position = vc->start() + s;
 378
 379                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
 380                 /* Here we are converting from time (in the DCP) to a frame number in the content.
 381                    Hence we need to use the DCP's frame rate and the double/skip correction, not
 382                    the source's rate.
 383                 */
 384                 VideoContent::Frame f = s * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
 385                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 386         }
 387
 388         _video_position = _audio_position = t;
 389
 390         /* XXX: don't seek audio because we don't need to... */
 391 }
 392
 393 void
 394 Player::setup_pieces ()
 395 {
 396         list<shared_ptr<Piece> > old_pieces = _pieces;
 397
 398         _pieces.clear ();
 399
 400         ContentList content = _playlist->content ();
 401         sort (content.begin(), content.end(), ContentSorter ());
 402
 403         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 404
 405                 shared_ptr<Piece> piece (new Piece (*i));
 406
 407                 /* XXX: into content? */
 408
 409                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 410                 if (fc) {
 411                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 412
 413                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 414                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 415                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
 416
 417                         piece->decoder = fd;
 418                 }
 419
 420                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
 421                 if (ic) {
 422                         shared_ptr<StillImageDecoder> id;
 423
 424                         /* See if we can re-use an old StillImageDecoder */
 425                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 426                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
 427                                 if (imd && imd->content() == ic) {
 428                                         id = imd;
 429                                 }
 430                         }
 431
 432                         if (!id) {
 433                                 id.reset (new StillImageDecoder (_film, ic));
 434                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 435                         }
 436
 437                         piece->decoder = id;
 438                 }
 439
 440                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
 441                 if (mc) {
 442                         shared_ptr<MovingImageDecoder> md;
 443
 444                         if (!md) {
 445                                 md.reset (new MovingImageDecoder (_film, mc));
 446                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 447                         }
 448
 449                         piece->decoder = md;
 450                 }
 451
 452                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 453                 if (sc) {
 454                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 455                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 456
 457                         piece->decoder = sd;
 458                 }
 459
 460                 _pieces.push_back (piece);
 461         }
 462
 463 #ifdef DEBUG_PLAYER
 464         cout << "=== Player setup:\n";
 465         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 466                 cout << *(i->get()) << "\n";
 467         }
 468 #endif
 469 }
 470
 471 void
 472 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 473 {
 474         shared_ptr<Content> c = w.lock ();
 475         if (!c) {
 476                 return;
 477         }
 478
 479         if (
 480                 property == ContentProperty::START || property == ContentProperty::LENGTH ||
 481                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_RATIO
 482                 ) {
 483
 484                 _have_valid_pieces = false;
 485                 Changed (frequent);
 486
 487         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
 488                 update_subtitle ();
 489                 Changed (frequent);
 490         } else if (property == VideoContentProperty::VIDEO_FRAME_TYPE) {
 491                 Changed (frequent);
 492         }
 493 }
 494
 495 void
 496 Player::playlist_changed ()
 497 {
 498         _have_valid_pieces = false;
 499         Changed (false);
 500 }
 501
 502 void
 503 Player::set_video_container_size (libdcp::Size s)
 504 {
 505         _video_container_size = s;
 506         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 507         _black_frame->make_black ();
 508 }
 509
 510 shared_ptr<Resampler>
 511 Player::resampler (shared_ptr<AudioContent> c, bool create)
 512 {
 513         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 514         if (i != _resamplers.end ()) {
 515                 return i->second;
 516         }
 517
 518         if (!create) {
 519                 return shared_ptr<Resampler> ();
 520         }
 521
 522         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 523         _resamplers[c] = r;
 524         return r;
 525 }
 526
 527 void
 528 Player::emit_black ()
 529 {
 530 #ifdef DCPOMATIC_DEBUG
 531         _last_video.reset ();
 532 #endif
 533
 534         /* XXX: use same here */
 535         Video (_black_frame, EYES_BOTH, false, _video_position);
 536         _video_position += _film->video_frames_to_time (1);
 537 }
 538
 539 void
 540 Player::emit_silence (OutputAudioFrame most)
 541 {
 542         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 543         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 544         silence->make_silent ();
 545         Audio (silence, _audio_position);
 546         _audio_position += _film->audio_frames_to_time (N);
 547 }
 548
 549 void
 550 Player::film_changed (Film::Property p)
 551 {
 552         /* Here we should notice Film properties that affect our output, and
 553            alert listeners that our output now would be different to how it was
 554            last time we were run.
 555         */
 556
 557         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 558                 Changed (false);
 559         }
 560 }
 561
 562 void
 563 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 564 {
 565         _in_subtitle.piece = weak_piece;
 566         _in_subtitle.image = image;
 567         _in_subtitle.rect = rect;
 568         _in_subtitle.from = from;
 569         _in_subtitle.to = to;
 570
 571         update_subtitle ();
 572 }
 573
 574 void
 575 Player::update_subtitle ()
 576 {
 577         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
 578         if (!piece) {
 579                 return;
 580         }
 581
 582         if (!_in_subtitle.image) {
 583                 _out_subtitle.image.reset ();
 584                 return;
 585         }
 586
 587         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 588         assert (sc);
 589
 590         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
 591         libdcp::Size scaled_size;
 592
 593         in_rect.y += sc->subtitle_offset ();
 594
 595         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 596         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
 597         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
 598
 599         /* Then we need a corrective translation, consisting of two parts:
 600          *
 601          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 602          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 603          *
 604          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 605          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 606          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 607          *
 608          * Combining these two translations gives these expressions.
 609          */
 610
 611         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
 612         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
 613
 614         _out_subtitle.image = _in_subtitle.image->scale (libdcp::Size (scaled_size.width, scaled_size.height), Scaler::from_id ("bicubic"), true);
 615         _out_subtitle.from = _in_subtitle.from + piece->content->start ();
 616         _out_subtitle.to = _in_subtitle.to + piece->content->start ();
 617 }