src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "image_proxy.h"
  34 #include "ratio.h"
  35 #include "resampler.h"
  36 #include "log.h"
  37 #include "scaler.h"
  38 #include "player_video_frame.h"
  39 #include "frame_rate_change.h"
  40
  41 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
  42
  43 using std::list;
  44 using std::cout;
  45 using std::min;
  46 using std::max;
  47 using std::vector;
  48 using std::pair;
  49 using std::map;
  50 using boost::shared_ptr;
  51 using boost::weak_ptr;
  52 using boost::dynamic_pointer_cast;
  53
  54 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  55         : _film (f)
  56         , _playlist (p)
  57         , _video (true)
  58         , _audio (true)
  59         , _have_valid_pieces (false)
  60         , _video_position (0)
  61         , _audio_position (0)
  62         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  63         , _last_emit_was_black (false)
  64 {
  65         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  66         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  67         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  68         set_video_container_size (_film->frame_size ());
  69 }
  70
  71 void
  72 Player::disable_video ()
  73 {
  74         _video = false;
  75 }
  76
  77 void
  78 Player::disable_audio ()
  79 {
  80         _audio = false;
  81 }
  82
  83 bool
  84 Player::pass ()
  85 {
  86         if (!_have_valid_pieces) {
  87                 setup_pieces ();
  88         }
  89
  90         Time earliest_t = TIME_MAX;
  91         shared_ptr<Piece> earliest;
  92         enum {
  93                 VIDEO,
  94                 AUDIO
  95         } type = VIDEO;
  96
  97         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
  98                 if ((*i)->decoder->done () || (*i)->content->length_after_trim() == 0) {
  99                         continue;
 100                 }
 101
 102                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
 103                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 104
 105                 if (_video && vd) {
 106                         if ((*i)->video_position < earliest_t) {
 107                                 earliest_t = (*i)->video_position;
 108                                 earliest = *i;
 109                                 type = VIDEO;
 110                         }
 111                 }
 112
 113                 if (_audio && ad && ad->has_audio ()) {
 114                         if ((*i)->audio_position < earliest_t) {
 115                                 earliest_t = (*i)->audio_position;
 116                                 earliest = *i;
 117                                 type = AUDIO;
 118                         }
 119                 }
 120         }
 121
 122         if (!earliest) {
 123                 flush ();
 124                 return true;
 125         }
 126
 127         switch (type) {
 128         case VIDEO:
 129                 if (earliest_t > _video_position) {
 130                         emit_black ();
 131                 } else {
 132                         if (earliest->repeating ()) {
 133                                 earliest->repeat (this);
 134                         } else {
 135                                 earliest->decoder->pass ();
 136                         }
 137                 }
 138                 break;
 139
 140         case AUDIO:
 141                 if (earliest_t > _audio_position) {
 142                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 143                 } else {
 144                         earliest->decoder->pass ();
 145
 146                         if (earliest->decoder->done()) {
 147                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 148                                 assert (ac);
 149                                 shared_ptr<Resampler> re = resampler (ac, false);
 150                                 if (re) {
 151                                         shared_ptr<const AudioBuffers> b = re->flush ();
 152                                         if (b->frames ()) {
 153                                                 process_audio (
 154                                                         earliest,
 155                                                         b,
 156                                                         ac->audio_length() * ac->output_audio_frame_rate() / ac->content_audio_frame_rate(),
 157                                                         true
 158                                                         );
 159                                         }
 160                                 }
 161                         }
 162                 }
 163                 break;
 164         }
 165
 166         if (_audio) {
 167                 boost::optional<Time> audio_done_up_to;
 168                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 169                         if ((*i)->decoder->done ()) {
 170                                 continue;
 171                         }
 172
 173                         shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 174                         if (ad && ad->has_audio ()) {
 175                                 audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position);
 176                         }
 177                 }
 178
 179                 if (audio_done_up_to) {
 180                         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to.get ());
 181                         Audio (tb.audio, tb.time);
 182                         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 183                 }
 184         }
 185
 186         return false;
 187 }
 188
 189 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
 190 void
 191 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const ImageProxy> image, Eyes eyes, Part part, bool same, VideoContent::Frame frame, Time extra)
 192 {
 193         /* Keep a note of what came in so that we can repeat it if required */
 194         _last_incoming_video.weak_piece = weak_piece;
 195         _last_incoming_video.image = image;
 196         _last_incoming_video.eyes = eyes;
 197         _last_incoming_video.part = part;
 198         _last_incoming_video.same = same;
 199         _last_incoming_video.frame = frame;
 200         _last_incoming_video.extra = extra;
 201
 202         shared_ptr<Piece> piece = weak_piece.lock ();
 203         if (!piece) {
 204                 return;
 205         }
 206
 207         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 208         assert (content);
 209
 210         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
 211         if (frc.skip && (frame % 2) == 1) {
 212                 return;
 213         }
 214
 215         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 216         if (content->trimmed (relative_time)) {
 217                 return;
 218         }
 219
 220         Time const time = content->position() + relative_time + extra - content->trim_start ();
 221         libdcp::Size const image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 222
 223         shared_ptr<PlayerVideoFrame> pi (
 224                 new PlayerVideoFrame (
 225                         image,
 226                         content->crop(),
 227                         image_size,
 228                         _video_container_size,
 229                         _film->scaler(),
 230                         eyes,
 231                         part,
 232                         content->colour_conversion()
 233                         )
 234                 );
 235
 236         if (_film->with_subtitles ()) {
 237                 for (list<Subtitle>::const_iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 238                         if (i->covers (time)) {
 239                                 /* This may be true for more than one of _subtitles, but the last (latest-starting)
 240                                    one is the one we want to use, so that's ok.
 241                                 */
 242                                 Position<int> const container_offset (
 243                                         (_video_container_size.width - image_size.width) / 2,
 244                                         (_video_container_size.height - image_size.width) / 2
 245                                         );
 246
 247                                 pi->set_subtitle (i->out_image(), i->out_position() + container_offset);
 248                         }
 249                 }
 250         }
 251
 252         /* Clear out old subtitles */
 253         for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ) {
 254                 list<Subtitle>::iterator j = i;
 255                 ++j;
 256
 257                 if (i->ends_before (time)) {
 258                         _subtitles.erase (i);
 259                 }
 260
 261                 i = j;
 262         }
 263
 264 #ifdef DCPOMATIC_DEBUG
 265         _last_video = piece->content;
 266 #endif
 267
 268         Video (pi, same, time);
 269
 270         _last_emit_was_black = false;
 271         _video_position = piece->video_position = (time + TIME_HZ / _film->video_frame_rate());
 272
 273         if (frc.repeat > 1 && !piece->repeating ()) {
 274                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
 275         }
 276 }
 277
 278 /** @param already_resampled true if this data has already been through the chain up to the resampler */
 279 void
 280 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame, bool already_resampled)
 281 {
 282         shared_ptr<Piece> piece = weak_piece.lock ();
 283         if (!piece) {
 284                 return;
 285         }
 286
 287         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 288         assert (content);
 289
 290         if (!already_resampled) {
 291                 /* Gain */
 292                 if (content->audio_gain() != 0) {
 293                         shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 294                         gain->apply_gain (content->audio_gain ());
 295                         audio = gain;
 296                 }
 297
 298                 /* Resample */
 299                 if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 300                         shared_ptr<Resampler> r = resampler (content, true);
 301                         pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 302                         audio = ro.first;
 303                         frame = ro.second;
 304                 }
 305         }
 306
 307         Time const relative_time = _film->audio_frames_to_time (frame);
 308
 309         if (content->trimmed (relative_time)) {
 310                 return;
 311         }
 312
 313         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
 314
 315         /* Remap channels */
 316         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 317         dcp_mapped->make_silent ();
 318
 319         AudioMapping map = content->audio_mapping ();
 320         for (int i = 0; i < map.content_channels(); ++i) {
 321                 for (int j = 0; j < _film->audio_channels(); ++j) {
 322                         if (map.get (i, static_cast<libdcp::Channel> (j)) > 0) {
 323                                 dcp_mapped->accumulate_channel (
 324                                         audio.get(),
 325                                         i,
 326                                         static_cast<libdcp::Channel> (j),
 327                                         map.get (i, static_cast<libdcp::Channel> (j))
 328                                         );
 329                         }
 330                 }
 331         }
 332
 333         audio = dcp_mapped;
 334
 335         /* We must cut off anything that comes before the start of all time */
 336         if (time < 0) {
 337                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 338                 if (frames >= audio->frames ()) {
 339                         return;
 340                 }
 341
 342                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 343                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 344
 345                 audio = trimmed;
 346                 time = 0;
 347         }
 348
 349         _audio_merger.push (audio, time);
 350         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 351 }
 352
 353 void
 354 Player::flush ()
 355 {
 356         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 357         if (_audio && tb.audio) {
 358                 Audio (tb.audio, tb.time);
 359                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 360         }
 361
 362         while (_video && _video_position < _audio_position) {
 363                 emit_black ();
 364         }
 365
 366         while (_audio && _audio_position < _video_position) {
 367                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 368         }
 369
 370 }
 371
 372 /** Seek so that the next pass() will yield (approximately) the requested frame.
 373  *  Pass accurate = true to try harder to get close to the request.
 374  *  @return true on error
 375  */
 376 void
 377 Player::seek (Time t, bool accurate)
 378 {
 379         if (!_have_valid_pieces) {
 380                 setup_pieces ();
 381         }
 382
 383         if (_pieces.empty ()) {
 384                 return;
 385         }
 386
 387         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 388                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 389                 if (!vc) {
 390                         continue;
 391                 }
 392
 393                 /* s is the offset of t from the start position of this content */
 394                 Time s = t - vc->position ();
 395                 s = max (static_cast<Time> (0), s);
 396                 s = min (vc->length_after_trim(), s);
 397
 398                 /* Hence set the piece positions to the `global' time */
 399                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 400
 401                 /* And seek the decoder */
 402                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
 403                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
 404                         );
 405
 406                 (*i)->reset_repeat ();
 407         }
 408
 409         _video_position = _audio_position = t;
 410
 411         /* XXX: don't seek audio because we don't need to... */
 412 }
 413
 414 void
 415 Player::setup_pieces ()
 416 {
 417         list<shared_ptr<Piece> > old_pieces = _pieces;
 418
 419         _pieces.clear ();
 420
 421         ContentList content = _playlist->content ();
 422         sort (content.begin(), content.end(), ContentSorter ());
 423
 424         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 425
 426                 if (!(*i)->paths_valid ()) {
 427                         continue;
 428                 }
 429
 430                 shared_ptr<Piece> piece (new Piece (*i));
 431
 432                 /* XXX: into content? */
 433
 434                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 435                 if (fc) {
 436                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 437
 438                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, _5, 0));
 439                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2, false));
 440                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
 441
 442                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
 443                         piece->decoder = fd;
 444                 }
 445
 446                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 447                 if (ic) {
 448                         bool reusing = false;
 449
 450                         /* See if we can re-use an old ImageDecoder */
 451                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 452                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 453                                 if (imd && imd->content() == ic) {
 454                                         piece = *j;
 455                                         reusing = true;
 456                                 }
 457                         }
 458
 459                         if (!reusing) {
 460                                 shared_ptr<ImageDecoder> id (new ImageDecoder (_film, ic));
 461                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, _5, 0));
 462                                 piece->decoder = id;
 463                         }
 464                 }
 465
 466                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 467                 if (sc) {
 468                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 469                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2, false));
 470
 471                         piece->decoder = sd;
 472                 }
 473
 474                 _pieces.push_back (piece);
 475         }
 476
 477         _have_valid_pieces = true;
 478 }
 479
 480 void
 481 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 482 {
 483         shared_ptr<Content> c = w.lock ();
 484         if (!c) {
 485                 return;
 486         }
 487
 488         if (
 489                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 490                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 491                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 492                 ) {
 493
 494                 _have_valid_pieces = false;
 495                 Changed (frequent);
 496
 497         } else if (
 498                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 499                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 500                 property == SubtitleContentProperty::SUBTITLE_SCALE
 501                 ) {
 502
 503                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 504                         i->update (_film, _video_container_size);
 505                 }
 506
 507                 Changed (frequent);
 508
 509         } else if (
 510                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_SCALE ||
 511                 property == VideoContentProperty::VIDEO_FRAME_RATE
 512                 ) {
 513
 514                 Changed (frequent);
 515
 516         } else if (property == ContentProperty::PATH) {
 517
 518                 _have_valid_pieces = false;
 519                 Changed (frequent);
 520         }
 521 }
 522
 523 void
 524 Player::playlist_changed ()
 525 {
 526         _have_valid_pieces = false;
 527         Changed (false);
 528 }
 529
 530 void
 531 Player::set_video_container_size (libdcp::Size s)
 532 {
 533         _video_container_size = s;
 534
 535         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 536         im->make_black ();
 537
 538         _black_frame.reset (
 539                 new PlayerVideoFrame (
 540                         shared_ptr<ImageProxy> (new RawImageProxy (im, _film->log ())),
 541                         Crop(),
 542                         _video_container_size,
 543                         _video_container_size,
 544                         Scaler::from_id ("bicubic"),
 545                         EYES_BOTH,
 546                         PART_WHOLE,
 547                         ColourConversion ()
 548                         )
 549                 );
 550 }
 551
 552 shared_ptr<Resampler>
 553 Player::resampler (shared_ptr<AudioContent> c, bool create)
 554 {
 555         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 556         if (i != _resamplers.end ()) {
 557                 return i->second;
 558         }
 559
 560         if (!create) {
 561                 return shared_ptr<Resampler> ();
 562         }
 563
 564         LOG_GENERAL (
 565                 "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
 566                 );
 567
 568         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 569         _resamplers[c] = r;
 570         return r;
 571 }
 572
 573 void
 574 Player::emit_black ()
 575 {
 576 #ifdef DCPOMATIC_DEBUG
 577         _last_video.reset ();
 578 #endif
 579
 580         Video (_black_frame, _last_emit_was_black, _video_position);
 581         _video_position += _film->video_frames_to_time (1);
 582         _last_emit_was_black = true;
 583 }
 584
 585 void
 586 Player::emit_silence (OutputAudioFrame most)
 587 {
 588         if (most == 0) {
 589                 return;
 590         }
 591
 592         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 593         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 594         silence->make_silent ();
 595         Audio (silence, _audio_position);
 596         _audio_position += _film->audio_frames_to_time (N);
 597 }
 598
 599 void
 600 Player::film_changed (Film::Property p)
 601 {
 602         /* Here we should notice Film properties that affect our output, and
 603            alert listeners that our output now would be different to how it was
 604            last time we were run.
 605         */
 606
 607         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 608                 Changed (false);
 609         }
 610 }
 611
 612 void
 613 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 614 {
 615         if (!image) {
 616                 /* A null image means that we should stop any current subtitles at `from' */
 617                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 618                         i->set_stop (from);
 619                 }
 620         } else {
 621                 _subtitles.push_back (Subtitle (_film, _video_container_size, weak_piece, image, rect, from, to));
 622         }
 623 }
 624
 625 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 626  *  @return false if this could not be done.
 627  */
 628 bool
 629 Player::repeat_last_video ()
 630 {
 631         if (!_last_incoming_video.image || !_have_valid_pieces) {
 632                 return false;
 633         }
 634
 635         process_video (
 636                 _last_incoming_video.weak_piece,
 637                 _last_incoming_video.image,
 638                 _last_incoming_video.eyes,
 639                 _last_incoming_video.part,
 640                 _last_incoming_video.same,
 641                 _last_incoming_video.frame,
 642                 _last_incoming_video.extra
 643                 );
 644
 645         return true;
 646 }