src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "ratio.h"
  34 #include "resampler.h"
  35 #include "log.h"
  36 #include "scaler.h"
  37 #include "player_video_frame.h"
  38
  39 using std::list;
  40 using std::cout;
  41 using std::min;
  42 using std::max;
  43 using std::vector;
  44 using std::pair;
  45 using std::map;
  46 using boost::shared_ptr;
  47 using boost::weak_ptr;
  48 using boost::dynamic_pointer_cast;
  49
  50 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  51         : _film (f)
  52         , _playlist (p)
  53         , _video (true)
  54         , _audio (true)
  55         , _have_valid_pieces (false)
  56         , _video_position (0)
  57         , _audio_position (0)
  58         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  59         , _last_emit_was_black (false)
  60 {
  61         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  62         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  63         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  64         set_video_container_size (_film->frame_size ());
  65 }
  66
  67 void
  68 Player::disable_video ()
  69 {
  70         _video = false;
  71 }
  72
  73 void
  74 Player::disable_audio ()
  75 {
  76         _audio = false;
  77 }
  78
  79 bool
  80 Player::pass ()
  81 {
  82         if (!_have_valid_pieces) {
  83                 setup_pieces ();
  84         }
  85
  86         Time earliest_t = TIME_MAX;
  87         shared_ptr<Piece> earliest;
  88         enum {
  89                 VIDEO,
  90                 AUDIO
  91         } type = VIDEO;
  92
  93         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
  94                 if ((*i)->decoder->done ()) {
  95                         continue;
  96                 }
  97
  98                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
  99                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 100
 101                 if (_video && vd) {
 102                         if ((*i)->video_position < earliest_t) {
 103                                 earliest_t = (*i)->video_position;
 104                                 earliest = *i;
 105                                 type = VIDEO;
 106                         }
 107                 }
 108
 109                 if (_audio && ad && ad->has_audio ()) {
 110                         if ((*i)->audio_position < earliest_t) {
 111                                 earliest_t = (*i)->audio_position;
 112                                 earliest = *i;
 113                                 type = AUDIO;
 114                         }
 115                 }
 116         }
 117
 118         if (!earliest) {
 119                 flush ();
 120                 return true;
 121         }
 122
 123         switch (type) {
 124         case VIDEO:
 125                 if (earliest_t > _video_position) {
 126                         emit_black ();
 127                 } else {
 128                         if (earliest->repeating ()) {
 129                                 earliest->repeat (this);
 130                         } else {
 131                                 earliest->decoder->pass ();
 132                         }
 133                 }
 134                 break;
 135
 136         case AUDIO:
 137                 if (earliest_t > _audio_position) {
 138                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 139                 } else {
 140                         earliest->decoder->pass ();
 141
 142                         if (earliest->decoder->done()) {
 143                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 144                                 assert (ac);
 145                                 shared_ptr<Resampler> re = resampler (ac, false);
 146                                 if (re) {
 147                                         shared_ptr<const AudioBuffers> b = re->flush ();
 148                                         if (b->frames ()) {
 149                                                 process_audio (earliest, b, ac->audio_length ());
 150                                         }
 151                                 }
 152                         }
 153                 }
 154                 break;
 155         }
 156
 157         if (_audio) {
 158                 boost::optional<Time> audio_done_up_to;
 159                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 160                         if ((*i)->decoder->done ()) {
 161                                 continue;
 162                         }
 163
 164                         shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 165                         if (ad && ad->has_audio ()) {
 166                                 audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position);
 167                         }
 168                 }
 169
 170                 if (audio_done_up_to) {
 171                         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to.get ());
 172                         Audio (tb.audio, tb.time);
 173                         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 174                 }
 175         }
 176
 177         return false;
 178 }
 179
 180 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
 181 void
 182 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame, Time extra)
 183 {
 184         /* Keep a note of what came in so that we can repeat it if required */
 185         _last_incoming_video.weak_piece = weak_piece;
 186         _last_incoming_video.image = image;
 187         _last_incoming_video.eyes = eyes;
 188         _last_incoming_video.same = same;
 189         _last_incoming_video.frame = frame;
 190         _last_incoming_video.extra = extra;
 191
 192         shared_ptr<Piece> piece = weak_piece.lock ();
 193         if (!piece) {
 194                 return;
 195         }
 196
 197         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 198         assert (content);
 199
 200         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 201         if (frc.skip && (frame % 2) == 1) {
 202                 return;
 203         }
 204
 205         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 206         if (content->trimmed (relative_time)) {
 207                 return;
 208         }
 209
 210         Time const time = content->position() + relative_time + extra - content->trim_start ();
 211         libdcp::Size const image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 212
 213         shared_ptr<PlayerVideoFrame> pi (
 214                 new PlayerVideoFrame (
 215                         image,
 216                         content->crop(),
 217                         image_size,
 218                         _video_container_size,
 219                         _film->scaler()
 220                         )
 221                 );
 222
 223         if (_film->with_subtitles ()) {
 224                 for (list<Subtitle>::const_iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 225                         if (i->covers (time)) {
 226                                 /* This may be true for more than one of _subtitles, but the last (latest-starting)
 227                                    one is the one we want to use, so that's ok.
 228                                 */
 229                                 Position<int> const container_offset (
 230                                         (_video_container_size.width - image_size.width) / 2,
 231                                         (_video_container_size.height - image_size.width) / 2
 232                                         );
 233
 234                                 pi->set_subtitle (i->out_image(), i->out_position() + container_offset);
 235                         }
 236                 }
 237         }
 238
 239         /* Clear out old subtitles */
 240         for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ) {
 241                 list<Subtitle>::iterator j = i;
 242                 ++j;
 243
 244                 if (i->ends_before (time)) {
 245                         _subtitles.erase (i);
 246                 }
 247
 248                 i = j;
 249         }
 250
 251 #ifdef DCPOMATIC_DEBUG
 252         _last_video = piece->content;
 253 #endif
 254
 255         Video (pi, eyes, content->colour_conversion(), same, time);
 256
 257         _last_emit_was_black = false;
 258         _video_position = piece->video_position = (time + TIME_HZ / _film->video_frame_rate());
 259
 260         if (frc.repeat > 1 && !piece->repeating ()) {
 261                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
 262         }
 263 }
 264
 265 void
 266 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 267 {
 268         shared_ptr<Piece> piece = weak_piece.lock ();
 269         if (!piece) {
 270                 return;
 271         }
 272
 273         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 274         assert (content);
 275
 276         /* Gain */
 277         if (content->audio_gain() != 0) {
 278                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 279                 gain->apply_gain (content->audio_gain ());
 280                 audio = gain;
 281         }
 282
 283         /* Resample */
 284         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 285                 shared_ptr<Resampler> r = resampler (content, true);
 286                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 287                 audio = ro.first;
 288                 frame = ro.second;
 289         }
 290
 291         Time const relative_time = _film->audio_frames_to_time (frame);
 292
 293         if (content->trimmed (relative_time)) {
 294                 return;
 295         }
 296
 297         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
 298
 299         /* Remap channels */
 300         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 301         dcp_mapped->make_silent ();
 302
 303         AudioMapping map = content->audio_mapping ();
 304         for (int i = 0; i < map.content_channels(); ++i) {
 305                 for (int j = 0; j < _film->audio_channels(); ++j) {
 306                         if (map.get (i, static_cast<libdcp::Channel> (j)) > 0) {
 307                                 dcp_mapped->accumulate_channel (
 308                                         audio.get(),
 309                                         i,
 310                                         static_cast<libdcp::Channel> (j),
 311                                         map.get (i, static_cast<libdcp::Channel> (j))
 312                                         );
 313                         }
 314                 }
 315         }
 316
 317         audio = dcp_mapped;
 318
 319         /* We must cut off anything that comes before the start of all time */
 320         if (time < 0) {
 321                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 322                 if (frames >= audio->frames ()) {
 323                         return;
 324                 }
 325
 326                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 327                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 328
 329                 audio = trimmed;
 330                 time = 0;
 331         }
 332
 333         _audio_merger.push (audio, time);
 334         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 335 }
 336
 337 void
 338 Player::flush ()
 339 {
 340         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 341         if (_audio && tb.audio) {
 342                 Audio (tb.audio, tb.time);
 343                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 344         }
 345
 346         while (_video && _video_position < _audio_position) {
 347                 emit_black ();
 348         }
 349
 350         while (_audio && _audio_position < _video_position) {
 351                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 352         }
 353
 354 }
 355
 356 /** Seek so that the next pass() will yield (approximately) the requested frame.
 357  *  Pass accurate = true to try harder to get close to the request.
 358  *  @return true on error
 359  */
 360 void
 361 Player::seek (Time t, bool accurate)
 362 {
 363         if (!_have_valid_pieces) {
 364                 setup_pieces ();
 365         }
 366
 367         if (_pieces.empty ()) {
 368                 return;
 369         }
 370
 371         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 372                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 373                 if (!vc) {
 374                         continue;
 375                 }
 376
 377                 /* s is the offset of t from the start position of this content */
 378                 Time s = t - vc->position ();
 379                 s = max (static_cast<Time> (0), s);
 380                 s = min (vc->length_after_trim(), s);
 381
 382                 /* Hence set the piece positions to the `global' time */
 383                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 384
 385                 /* And seek the decoder */
 386                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
 387                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
 388                         );
 389
 390                 (*i)->reset_repeat ();
 391         }
 392
 393         _video_position = _audio_position = t;
 394
 395         /* XXX: don't seek audio because we don't need to... */
 396 }
 397
 398 void
 399 Player::setup_pieces ()
 400 {
 401         list<shared_ptr<Piece> > old_pieces = _pieces;
 402
 403         _pieces.clear ();
 404
 405         ContentList content = _playlist->content ();
 406         sort (content.begin(), content.end(), ContentSorter ());
 407
 408         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 409
 410                 if (!(*i)->paths_valid ()) {
 411                         continue;
 412                 }
 413
 414                 shared_ptr<Piece> piece (new Piece (*i));
 415
 416                 /* XXX: into content? */
 417
 418                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 419                 if (fc) {
 420                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 421
 422                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
 423                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 424                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
 425
 426                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
 427                         piece->decoder = fd;
 428                 }
 429
 430                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 431                 if (ic) {
 432                         bool reusing = false;
 433
 434                         /* See if we can re-use an old ImageDecoder */
 435                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 436                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 437                                 if (imd && imd->content() == ic) {
 438                                         piece = *j;
 439                                         reusing = true;
 440                                 }
 441                         }
 442
 443                         if (!reusing) {
 444                                 shared_ptr<ImageDecoder> id (new ImageDecoder (_film, ic));
 445                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, 0));
 446                                 piece->decoder = id;
 447                         }
 448                 }
 449
 450                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 451                 if (sc) {
 452                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 453                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 454
 455                         piece->decoder = sd;
 456                 }
 457
 458                 _pieces.push_back (piece);
 459         }
 460
 461         _have_valid_pieces = true;
 462 }
 463
 464 void
 465 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 466 {
 467         shared_ptr<Content> c = w.lock ();
 468         if (!c) {
 469                 return;
 470         }
 471
 472         if (
 473                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 474                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 475                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 476                 ) {
 477
 478                 _have_valid_pieces = false;
 479                 Changed (frequent);
 480
 481         } else if (
 482                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 483                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 484                 property == SubtitleContentProperty::SUBTITLE_SCALE
 485                 ) {
 486
 487                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 488                         i->update (_film, _video_container_size);
 489                 }
 490
 491                 Changed (frequent);
 492
 493         } else if (
 494                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_SCALE ||
 495                 property == VideoContentProperty::VIDEO_FRAME_RATE
 496                 ) {
 497
 498                 Changed (frequent);
 499
 500         } else if (property == ContentProperty::PATH) {
 501
 502                 _have_valid_pieces = false;
 503                 Changed (frequent);
 504         }
 505 }
 506
 507 void
 508 Player::playlist_changed ()
 509 {
 510         _have_valid_pieces = false;
 511         Changed (false);
 512 }
 513
 514 void
 515 Player::set_video_container_size (libdcp::Size s)
 516 {
 517         _video_container_size = s;
 518
 519         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 520         im->make_black ();
 521
 522         _black_frame.reset (
 523                 new PlayerVideoFrame (
 524                         im,
 525                         Crop(),
 526                         _video_container_size,
 527                         _video_container_size,
 528                         Scaler::from_id ("bicubic")
 529                         )
 530                 );
 531 }
 532
 533 shared_ptr<Resampler>
 534 Player::resampler (shared_ptr<AudioContent> c, bool create)
 535 {
 536         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 537         if (i != _resamplers.end ()) {
 538                 return i->second;
 539         }
 540
 541         if (!create) {
 542                 return shared_ptr<Resampler> ();
 543         }
 544
 545         _film->log()->log (
 546                 String::compose (
 547                         "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
 548                         )
 549                 );
 550
 551         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 552         _resamplers[c] = r;
 553         return r;
 554 }
 555
 556 void
 557 Player::emit_black ()
 558 {
 559 #ifdef DCPOMATIC_DEBUG
 560         _last_video.reset ();
 561 #endif
 562
 563         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
 564         _video_position += _film->video_frames_to_time (1);
 565         _last_emit_was_black = true;
 566 }
 567
 568 void
 569 Player::emit_silence (OutputAudioFrame most)
 570 {
 571         if (most == 0) {
 572                 return;
 573         }
 574
 575         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 576         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 577         silence->make_silent ();
 578         Audio (silence, _audio_position);
 579         _audio_position += _film->audio_frames_to_time (N);
 580 }
 581
 582 void
 583 Player::film_changed (Film::Property p)
 584 {
 585         /* Here we should notice Film properties that affect our output, and
 586            alert listeners that our output now would be different to how it was
 587            last time we were run.
 588         */
 589
 590         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 591                 Changed (false);
 592         }
 593 }
 594
 595 void
 596 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 597 {
 598         if (!image) {
 599                 /* A null image means that we should stop any current subtitles at `from' */
 600                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 601                         i->set_stop (from);
 602                 }
 603         } else {
 604                 _subtitles.push_back (Subtitle (_film, _video_container_size, weak_piece, image, rect, from, to));
 605         }
 606 }
 607
 608 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 609  *  @return false if this could not be done.
 610  */
 611 bool
 612 Player::repeat_last_video ()
 613 {
 614         if (!_last_incoming_video.image || !_have_valid_pieces) {
 615                 return false;
 616         }
 617
 618         process_video (
 619                 _last_incoming_video.weak_piece,
 620                 _last_incoming_video.image,
 621                 _last_incoming_video.eyes,
 622                 _last_incoming_video.same,
 623                 _last_incoming_video.frame,
 624                 _last_incoming_video.extra
 625                 );
 626
 627         return true;
 628 }