src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "image_proxy.h"
  34 #include "ratio.h"
  35 #include "resampler.h"
  36 #include "log.h"
  37 #include "scaler.h"
  38 #include "player_video_frame.h"
  39
  40 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
  41
  42 using std::list;
  43 using std::cout;
  44 using std::min;
  45 using std::max;
  46 using std::vector;
  47 using std::pair;
  48 using std::map;
  49 using boost::shared_ptr;
  50 using boost::weak_ptr;
  51 using boost::dynamic_pointer_cast;
  52
  53 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  54         : _film (f)
  55         , _playlist (p)
  56         , _video (true)
  57         , _audio (true)
  58         , _have_valid_pieces (false)
  59         , _video_position (0)
  60         , _audio_position (0)
  61         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  62         , _last_emit_was_black (false)
  63 {
  64         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  65         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  66         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  67         set_video_container_size (_film->frame_size ());
  68 }
  69
  70 void
  71 Player::disable_video ()
  72 {
  73         _video = false;
  74 }
  75
  76 void
  77 Player::disable_audio ()
  78 {
  79         _audio = false;
  80 }
  81
  82 bool
  83 Player::pass ()
  84 {
  85         if (!_have_valid_pieces) {
  86                 setup_pieces ();
  87         }
  88
  89         Time earliest_t = TIME_MAX;
  90         shared_ptr<Piece> earliest;
  91         enum {
  92                 VIDEO,
  93                 AUDIO
  94         } type = VIDEO;
  95
  96         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
  97                 if ((*i)->decoder->done ()) {
  98                         continue;
  99                 }
 100
 101                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
 102                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 103
 104                 if (_video && vd) {
 105                         if ((*i)->video_position < earliest_t) {
 106                                 earliest_t = (*i)->video_position;
 107                                 earliest = *i;
 108                                 type = VIDEO;
 109                         }
 110                 }
 111
 112                 if (_audio && ad && ad->has_audio ()) {
 113                         if ((*i)->audio_position < earliest_t) {
 114                                 earliest_t = (*i)->audio_position;
 115                                 earliest = *i;
 116                                 type = AUDIO;
 117                         }
 118                 }
 119         }
 120
 121         if (!earliest) {
 122                 flush ();
 123                 return true;
 124         }
 125
 126         switch (type) {
 127         case VIDEO:
 128                 if (earliest_t > _video_position) {
 129                         emit_black ();
 130                 } else {
 131                         if (earliest->repeating ()) {
 132                                 earliest->repeat (this);
 133                         } else {
 134                                 earliest->decoder->pass ();
 135                         }
 136                 }
 137                 break;
 138
 139         case AUDIO:
 140                 if (earliest_t > _audio_position) {
 141                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 142                 } else {
 143                         earliest->decoder->pass ();
 144
 145                         if (earliest->decoder->done()) {
 146                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 147                                 assert (ac);
 148                                 shared_ptr<Resampler> re = resampler (ac, false);
 149                                 if (re) {
 150                                         shared_ptr<const AudioBuffers> b = re->flush ();
 151                                         if (b->frames ()) {
 152                                                 process_audio (earliest, b, ac->audio_length ());
 153                                         }
 154                                 }
 155                         }
 156                 }
 157                 break;
 158         }
 159
 160         if (_audio) {
 161                 boost::optional<Time> audio_done_up_to;
 162                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 163                         if ((*i)->decoder->done ()) {
 164                                 continue;
 165                         }
 166
 167                         shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 168                         if (ad && ad->has_audio ()) {
 169                                 audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position);
 170                         }
 171                 }
 172
 173                 if (audio_done_up_to) {
 174                         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to.get ());
 175                         Audio (tb.audio, tb.time);
 176                         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 177                 }
 178         }
 179
 180         return false;
 181 }
 182
 183 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
 184 void
 185 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const ImageProxy> image, Eyes eyes, Part part, bool same, VideoContent::Frame frame, Time extra)
 186 {
 187         /* Keep a note of what came in so that we can repeat it if required */
 188         _last_incoming_video.weak_piece = weak_piece;
 189         _last_incoming_video.image = image;
 190         _last_incoming_video.eyes = eyes;
 191         _last_incoming_video.part = part;
 192         _last_incoming_video.same = same;
 193         _last_incoming_video.frame = frame;
 194         _last_incoming_video.extra = extra;
 195
 196         shared_ptr<Piece> piece = weak_piece.lock ();
 197         if (!piece) {
 198                 return;
 199         }
 200
 201         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 202         assert (content);
 203
 204         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 205         if (frc.skip && (frame % 2) == 1) {
 206                 return;
 207         }
 208
 209         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 210         if (content->trimmed (relative_time)) {
 211                 return;
 212         }
 213
 214         Time const time = content->position() + relative_time + extra - content->trim_start ();
 215         libdcp::Size const image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 216
 217         shared_ptr<PlayerVideoFrame> pi (
 218                 new PlayerVideoFrame (
 219                         image,
 220                         content->crop(),
 221                         image_size,
 222                         _video_container_size,
 223                         _film->scaler(),
 224                         eyes,
 225                         part,
 226                         content->colour_conversion()
 227                         )
 228                 );
 229
 230         if (_film->with_subtitles ()) {
 231                 for (list<Subtitle>::const_iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 232                         if (i->covers (time)) {
 233                                 /* This may be true for more than one of _subtitles, but the last (latest-starting)
 234                                    one is the one we want to use, so that's ok.
 235                                 */
 236                                 Position<int> const container_offset (
 237                                         (_video_container_size.width - image_size.width) / 2,
 238                                         (_video_container_size.height - image_size.width) / 2
 239                                         );
 240
 241                                 pi->set_subtitle (i->out_image(), i->out_position() + container_offset);
 242                         }
 243                 }
 244         }
 245
 246         /* Clear out old subtitles */
 247         for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ) {
 248                 list<Subtitle>::iterator j = i;
 249                 ++j;
 250
 251                 if (i->ends_before (time)) {
 252                         _subtitles.erase (i);
 253                 }
 254
 255                 i = j;
 256         }
 257
 258 #ifdef DCPOMATIC_DEBUG
 259         _last_video = piece->content;
 260 #endif
 261
 262         Video (pi, same, time);
 263
 264         _last_emit_was_black = false;
 265         _video_position = piece->video_position = (time + TIME_HZ / _film->video_frame_rate());
 266
 267         if (frc.repeat > 1 && !piece->repeating ()) {
 268                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
 269         }
 270 }
 271
 272 void
 273 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 274 {
 275         shared_ptr<Piece> piece = weak_piece.lock ();
 276         if (!piece) {
 277                 return;
 278         }
 279
 280         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 281         assert (content);
 282
 283         /* Gain */
 284         if (content->audio_gain() != 0) {
 285                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 286                 gain->apply_gain (content->audio_gain ());
 287                 audio = gain;
 288         }
 289
 290         /* Resample */
 291         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 292                 shared_ptr<Resampler> r = resampler (content, true);
 293                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 294                 audio = ro.first;
 295                 frame = ro.second;
 296         }
 297
 298         Time const relative_time = _film->audio_frames_to_time (frame);
 299
 300         if (content->trimmed (relative_time)) {
 301                 return;
 302         }
 303
 304         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
 305
 306         /* Remap channels */
 307         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 308         dcp_mapped->make_silent ();
 309
 310         AudioMapping map = content->audio_mapping ();
 311         for (int i = 0; i < map.content_channels(); ++i) {
 312                 for (int j = 0; j < _film->audio_channels(); ++j) {
 313                         if (map.get (i, static_cast<libdcp::Channel> (j)) > 0) {
 314                                 dcp_mapped->accumulate_channel (
 315                                         audio.get(),
 316                                         i,
 317                                         static_cast<libdcp::Channel> (j),
 318                                         map.get (i, static_cast<libdcp::Channel> (j))
 319                                         );
 320                         }
 321                 }
 322         }
 323
 324         audio = dcp_mapped;
 325
 326         /* We must cut off anything that comes before the start of all time */
 327         if (time < 0) {
 328                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 329                 if (frames >= audio->frames ()) {
 330                         return;
 331                 }
 332
 333                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 334                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 335
 336                 audio = trimmed;
 337                 time = 0;
 338         }
 339
 340         _audio_merger.push (audio, time);
 341         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 342 }
 343
 344 void
 345 Player::flush ()
 346 {
 347         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 348         if (_audio && tb.audio) {
 349                 Audio (tb.audio, tb.time);
 350                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 351         }
 352
 353         while (_video && _video_position < _audio_position) {
 354                 emit_black ();
 355         }
 356
 357         while (_audio && _audio_position < _video_position) {
 358                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 359         }
 360
 361 }
 362
 363 /** Seek so that the next pass() will yield (approximately) the requested frame.
 364  *  Pass accurate = true to try harder to get close to the request.
 365  *  @return true on error
 366  */
 367 void
 368 Player::seek (Time t, bool accurate)
 369 {
 370         if (!_have_valid_pieces) {
 371                 setup_pieces ();
 372         }
 373
 374         if (_pieces.empty ()) {
 375                 return;
 376         }
 377
 378         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 379                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 380                 if (!vc) {
 381                         continue;
 382                 }
 383
 384                 /* s is the offset of t from the start position of this content */
 385                 Time s = t - vc->position ();
 386                 s = max (static_cast<Time> (0), s);
 387                 s = min (vc->length_after_trim(), s);
 388
 389                 /* Hence set the piece positions to the `global' time */
 390                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 391
 392                 /* And seek the decoder */
 393                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
 394                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
 395                         );
 396
 397                 (*i)->reset_repeat ();
 398         }
 399
 400         _video_position = _audio_position = t;
 401
 402         /* XXX: don't seek audio because we don't need to... */
 403 }
 404
 405 void
 406 Player::setup_pieces ()
 407 {
 408         list<shared_ptr<Piece> > old_pieces = _pieces;
 409
 410         _pieces.clear ();
 411
 412         ContentList content = _playlist->content ();
 413         sort (content.begin(), content.end(), ContentSorter ());
 414
 415         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 416
 417                 if (!(*i)->paths_valid ()) {
 418                         continue;
 419                 }
 420
 421                 shared_ptr<Piece> piece (new Piece (*i));
 422
 423                 /* XXX: into content? */
 424
 425                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 426                 if (fc) {
 427                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 428
 429                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, _5, 0));
 430                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 431                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
 432
 433                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
 434                         piece->decoder = fd;
 435                 }
 436
 437                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 438                 if (ic) {
 439                         bool reusing = false;
 440
 441                         /* See if we can re-use an old ImageDecoder */
 442                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 443                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 444                                 if (imd && imd->content() == ic) {
 445                                         piece = *j;
 446                                         reusing = true;
 447                                 }
 448                         }
 449
 450                         if (!reusing) {
 451                                 shared_ptr<ImageDecoder> id (new ImageDecoder (_film, ic));
 452                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, _5, 0));
 453                                 piece->decoder = id;
 454                         }
 455                 }
 456
 457                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 458                 if (sc) {
 459                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 460                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 461
 462                         piece->decoder = sd;
 463                 }
 464
 465                 _pieces.push_back (piece);
 466         }
 467
 468         _have_valid_pieces = true;
 469 }
 470
 471 void
 472 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 473 {
 474         shared_ptr<Content> c = w.lock ();
 475         if (!c) {
 476                 return;
 477         }
 478
 479         if (
 480                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 481                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 482                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 483                 ) {
 484
 485                 _have_valid_pieces = false;
 486                 Changed (frequent);
 487
 488         } else if (
 489                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 490                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 491                 property == SubtitleContentProperty::SUBTITLE_SCALE
 492                 ) {
 493
 494                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 495                         i->update (_film, _video_container_size);
 496                 }
 497
 498                 Changed (frequent);
 499
 500         } else if (
 501                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_SCALE ||
 502                 property == VideoContentProperty::VIDEO_FRAME_RATE
 503                 ) {
 504
 505                 Changed (frequent);
 506
 507         } else if (property == ContentProperty::PATH) {
 508
 509                 _have_valid_pieces = false;
 510                 Changed (frequent);
 511         }
 512 }
 513
 514 void
 515 Player::playlist_changed ()
 516 {
 517         _have_valid_pieces = false;
 518         Changed (false);
 519 }
 520
 521 void
 522 Player::set_video_container_size (libdcp::Size s)
 523 {
 524         _video_container_size = s;
 525
 526         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 527         im->make_black ();
 528
 529         _black_frame.reset (
 530                 new PlayerVideoFrame (
 531                         shared_ptr<ImageProxy> (new RawImageProxy (im, _film->log ())),
 532                         Crop(),
 533                         _video_container_size,
 534                         _video_container_size,
 535                         Scaler::from_id ("bicubic"),
 536                         EYES_BOTH,
 537                         PART_WHOLE,
 538                         ColourConversion ()
 539                         )
 540                 );
 541 }
 542
 543 shared_ptr<Resampler>
 544 Player::resampler (shared_ptr<AudioContent> c, bool create)
 545 {
 546         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 547         if (i != _resamplers.end ()) {
 548                 return i->second;
 549         }
 550
 551         if (!create) {
 552                 return shared_ptr<Resampler> ();
 553         }
 554
 555         LOG_GENERAL (
 556                 "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
 557                 );
 558
 559         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 560         _resamplers[c] = r;
 561         return r;
 562 }
 563
 564 void
 565 Player::emit_black ()
 566 {
 567 #ifdef DCPOMATIC_DEBUG
 568         _last_video.reset ();
 569 #endif
 570
 571         Video (_black_frame, _last_emit_was_black, _video_position);
 572         _video_position += _film->video_frames_to_time (1);
 573         _last_emit_was_black = true;
 574 }
 575
 576 void
 577 Player::emit_silence (OutputAudioFrame most)
 578 {
 579         if (most == 0) {
 580                 return;
 581         }
 582
 583         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 584         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 585         silence->make_silent ();
 586         Audio (silence, _audio_position);
 587         _audio_position += _film->audio_frames_to_time (N);
 588 }
 589
 590 void
 591 Player::film_changed (Film::Property p)
 592 {
 593         /* Here we should notice Film properties that affect our output, and
 594            alert listeners that our output now would be different to how it was
 595            last time we were run.
 596         */
 597
 598         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 599                 Changed (false);
 600         }
 601 }
 602
 603 void
 604 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 605 {
 606         if (!image) {
 607                 /* A null image means that we should stop any current subtitles at `from' */
 608                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 609                         i->set_stop (from);
 610                 }
 611         } else {
 612                 _subtitles.push_back (Subtitle (_film, _video_container_size, weak_piece, image, rect, from, to));
 613         }
 614 }
 615
 616 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 617  *  @return false if this could not be done.
 618  */
 619 bool
 620 Player::repeat_last_video ()
 621 {
 622         if (!_last_incoming_video.image || !_have_valid_pieces) {
 623                 return false;
 624         }
 625
 626         process_video (
 627                 _last_incoming_video.weak_piece,
 628                 _last_incoming_video.image,
 629                 _last_incoming_video.eyes,
 630                 _last_incoming_video.part,
 631                 _last_incoming_video.same,
 632                 _last_incoming_video.frame,
 633                 _last_incoming_video.extra
 634                 );
 635
 636         return true;
 637 }