src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "image_proxy.h"
  34 #include "ratio.h"
  35 #include "resampler.h"
  36 #include "log.h"
  37 #include "scaler.h"
  38 #include "player_video_frame.h"
  39
  40 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
  41
  42 using std::list;
  43 using std::cout;
  44 using std::min;
  45 using std::max;
  46 using std::vector;
  47 using std::pair;
  48 using std::map;
  49 using boost::shared_ptr;
  50 using boost::weak_ptr;
  51 using boost::dynamic_pointer_cast;
  52
  53 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  54         : _film (f)
  55         , _playlist (p)
  56         , _video (true)
  57         , _audio (true)
  58         , _have_valid_pieces (false)
  59         , _video_position (0)
  60         , _audio_position (0)
  61         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  62         , _last_emit_was_black (false)
  63 {
  64         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  65         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  66         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  67         set_video_container_size (_film->frame_size ());
  68 }
  69
  70 void
  71 Player::disable_video ()
  72 {
  73         _video = false;
  74 }
  75
  76 void
  77 Player::disable_audio ()
  78 {
  79         _audio = false;
  80 }
  81
  82 bool
  83 Player::pass ()
  84 {
  85         if (!_have_valid_pieces) {
  86                 setup_pieces ();
  87         }
  88
  89         Time earliest_t = TIME_MAX;
  90         shared_ptr<Piece> earliest;
  91         enum {
  92                 VIDEO,
  93                 AUDIO
  94         } type = VIDEO;
  95
  96         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
  97                 if ((*i)->decoder->done ()) {
  98                         continue;
  99                 }
 100
 101                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
 102                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 103
 104                 if (_video && vd) {
 105                         if ((*i)->video_position < earliest_t) {
 106                                 earliest_t = (*i)->video_position;
 107                                 earliest = *i;
 108                                 type = VIDEO;
 109                         }
 110                 }
 111
 112                 if (_audio && ad && ad->has_audio ()) {
 113                         if ((*i)->audio_position < earliest_t) {
 114                                 earliest_t = (*i)->audio_position;
 115                                 earliest = *i;
 116                                 type = AUDIO;
 117                         }
 118                 }
 119         }
 120
 121         if (!earliest) {
 122                 flush ();
 123                 return true;
 124         }
 125
 126         switch (type) {
 127         case VIDEO:
 128                 if (earliest_t > _video_position) {
 129                         emit_black ();
 130                 } else {
 131                         if (earliest->repeating ()) {
 132                                 earliest->repeat (this);
 133                         } else {
 134                                 earliest->decoder->pass ();
 135                         }
 136                 }
 137                 break;
 138
 139         case AUDIO:
 140                 if (earliest_t > _audio_position) {
 141                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 142                 } else {
 143                         earliest->decoder->pass ();
 144
 145                         if (earliest->decoder->done()) {
 146                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 147                                 assert (ac);
 148                                 shared_ptr<Resampler> re = resampler (ac, false);
 149                                 if (re) {
 150                                         shared_ptr<const AudioBuffers> b = re->flush ();
 151                                         if (b->frames ()) {
 152                                                 process_audio (earliest, b, ac->audio_length (), true);
 153                                         }
 154                                 }
 155                         }
 156                 }
 157                 break;
 158         }
 159
 160         if (_audio) {
 161                 boost::optional<Time> audio_done_up_to;
 162                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 163                         if ((*i)->decoder->done ()) {
 164                                 continue;
 165                         }
 166
 167                         shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 168                         if (ad && ad->has_audio ()) {
 169                                 audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position);
 170                         }
 171                 }
 172
 173                 if (audio_done_up_to) {
 174                         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to.get ());
 175                         Audio (tb.audio, tb.time);
 176                         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 177                 }
 178         }
 179
 180         return false;
 181 }
 182
 183 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
 184 void
 185 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const ImageProxy> image, Eyes eyes, Part part, bool same, VideoContent::Frame frame, Time extra)
 186 {
 187         /* Keep a note of what came in so that we can repeat it if required */
 188         _last_incoming_video.weak_piece = weak_piece;
 189         _last_incoming_video.image = image;
 190         _last_incoming_video.eyes = eyes;
 191         _last_incoming_video.part = part;
 192         _last_incoming_video.same = same;
 193         _last_incoming_video.frame = frame;
 194         _last_incoming_video.extra = extra;
 195
 196         shared_ptr<Piece> piece = weak_piece.lock ();
 197         if (!piece) {
 198                 return;
 199         }
 200
 201         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 202         assert (content);
 203
 204         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 205         if (frc.skip && (frame % 2) == 1) {
 206                 return;
 207         }
 208
 209         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 210         if (content->trimmed (relative_time)) {
 211                 return;
 212         }
 213
 214         Time const time = content->position() + relative_time + extra - content->trim_start ();
 215         libdcp::Size const image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 216
 217         shared_ptr<PlayerVideoFrame> pi (
 218                 new PlayerVideoFrame (
 219                         image,
 220                         content->crop(),
 221                         image_size,
 222                         _video_container_size,
 223                         _film->scaler(),
 224                         eyes,
 225                         part,
 226                         content->colour_conversion()
 227                         )
 228                 );
 229
 230         if (_film->with_subtitles ()) {
 231                 for (list<Subtitle>::const_iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 232                         if (i->covers (time)) {
 233                                 /* This may be true for more than one of _subtitles, but the last (latest-starting)
 234                                    one is the one we want to use, so that's ok.
 235                                 */
 236                                 Position<int> const container_offset (
 237                                         (_video_container_size.width - image_size.width) / 2,
 238                                         (_video_container_size.height - image_size.width) / 2
 239                                         );
 240
 241                                 pi->set_subtitle (i->out_image(), i->out_position() + container_offset);
 242                         }
 243                 }
 244         }
 245
 246         /* Clear out old subtitles */
 247         for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ) {
 248                 list<Subtitle>::iterator j = i;
 249                 ++j;
 250
 251                 if (i->ends_before (time)) {
 252                         _subtitles.erase (i);
 253                 }
 254
 255                 i = j;
 256         }
 257
 258 #ifdef DCPOMATIC_DEBUG
 259         _last_video = piece->content;
 260 #endif
 261
 262         Video (pi, same, time);
 263
 264         _last_emit_was_black = false;
 265         _video_position = piece->video_position = (time + TIME_HZ / _film->video_frame_rate());
 266
 267         if (frc.repeat > 1 && !piece->repeating ()) {
 268                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
 269         }
 270 }
 271
 272 /** @param already_resampled true if this data has already been through the chain up to the resampler */
 273 void
 274 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame, bool already_resampled)
 275 {
 276         shared_ptr<Piece> piece = weak_piece.lock ();
 277         if (!piece) {
 278                 return;
 279         }
 280
 281         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 282         assert (content);
 283
 284         if (!already_resampled) {
 285                 /* Gain */
 286                 if (content->audio_gain() != 0) {
 287                         shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 288                         gain->apply_gain (content->audio_gain ());
 289                         audio = gain;
 290                 }
 291
 292                 /* Resample */
 293                 if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 294                         shared_ptr<Resampler> r = resampler (content, true);
 295                         pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 296                         audio = ro.first;
 297                         frame = ro.second;
 298                 }
 299         }
 300
 301         Time const relative_time = _film->audio_frames_to_time (frame);
 302
 303         if (content->trimmed (relative_time)) {
 304                 return;
 305         }
 306
 307         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
 308
 309         /* Remap channels */
 310         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 311         dcp_mapped->make_silent ();
 312
 313         AudioMapping map = content->audio_mapping ();
 314         for (int i = 0; i < map.content_channels(); ++i) {
 315                 for (int j = 0; j < _film->audio_channels(); ++j) {
 316                         if (map.get (i, static_cast<libdcp::Channel> (j)) > 0) {
 317                                 dcp_mapped->accumulate_channel (
 318                                         audio.get(),
 319                                         i,
 320                                         static_cast<libdcp::Channel> (j),
 321                                         map.get (i, static_cast<libdcp::Channel> (j))
 322                                         );
 323                         }
 324                 }
 325         }
 326
 327         audio = dcp_mapped;
 328
 329         /* We must cut off anything that comes before the start of all time */
 330         if (time < 0) {
 331                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 332                 if (frames >= audio->frames ()) {
 333                         return;
 334                 }
 335
 336                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 337                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 338
 339                 audio = trimmed;
 340                 time = 0;
 341         }
 342
 343         _audio_merger.push (audio, time);
 344         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 345 }
 346
 347 void
 348 Player::flush ()
 349 {
 350         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 351         if (_audio && tb.audio) {
 352                 Audio (tb.audio, tb.time);
 353                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 354         }
 355
 356         while (_video && _video_position < _audio_position) {
 357                 emit_black ();
 358         }
 359
 360         while (_audio && _audio_position < _video_position) {
 361                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 362         }
 363
 364 }
 365
 366 /** Seek so that the next pass() will yield (approximately) the requested frame.
 367  *  Pass accurate = true to try harder to get close to the request.
 368  *  @return true on error
 369  */
 370 void
 371 Player::seek (Time t, bool accurate)
 372 {
 373         if (!_have_valid_pieces) {
 374                 setup_pieces ();
 375         }
 376
 377         if (_pieces.empty ()) {
 378                 return;
 379         }
 380
 381         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 382                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 383                 if (!vc) {
 384                         continue;
 385                 }
 386
 387                 /* s is the offset of t from the start position of this content */
 388                 Time s = t - vc->position ();
 389                 s = max (static_cast<Time> (0), s);
 390                 s = min (vc->length_after_trim(), s);
 391
 392                 /* Hence set the piece positions to the `global' time */
 393                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 394
 395                 /* And seek the decoder */
 396                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
 397                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
 398                         );
 399
 400                 (*i)->reset_repeat ();
 401         }
 402
 403         _video_position = _audio_position = t;
 404
 405         /* XXX: don't seek audio because we don't need to... */
 406 }
 407
 408 void
 409 Player::setup_pieces ()
 410 {
 411         list<shared_ptr<Piece> > old_pieces = _pieces;
 412
 413         _pieces.clear ();
 414
 415         ContentList content = _playlist->content ();
 416         sort (content.begin(), content.end(), ContentSorter ());
 417
 418         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 419
 420                 if (!(*i)->paths_valid ()) {
 421                         continue;
 422                 }
 423
 424                 shared_ptr<Piece> piece (new Piece (*i));
 425
 426                 /* XXX: into content? */
 427
 428                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 429                 if (fc) {
 430                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 431
 432                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, _5, 0));
 433                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2, false));
 434                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
 435
 436                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
 437                         piece->decoder = fd;
 438                 }
 439
 440                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 441                 if (ic) {
 442                         bool reusing = false;
 443
 444                         /* See if we can re-use an old ImageDecoder */
 445                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 446                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 447                                 if (imd && imd->content() == ic) {
 448                                         piece = *j;
 449                                         reusing = true;
 450                                 }
 451                         }
 452
 453                         if (!reusing) {
 454                                 shared_ptr<ImageDecoder> id (new ImageDecoder (_film, ic));
 455                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, _5, 0));
 456                                 piece->decoder = id;
 457                         }
 458                 }
 459
 460                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 461                 if (sc) {
 462                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 463                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2, false));
 464
 465                         piece->decoder = sd;
 466                 }
 467
 468                 _pieces.push_back (piece);
 469         }
 470
 471         _have_valid_pieces = true;
 472 }
 473
 474 void
 475 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 476 {
 477         shared_ptr<Content> c = w.lock ();
 478         if (!c) {
 479                 return;
 480         }
 481
 482         if (
 483                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 484                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 485                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 486                 ) {
 487
 488                 _have_valid_pieces = false;
 489                 Changed (frequent);
 490
 491         } else if (
 492                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 493                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 494                 property == SubtitleContentProperty::SUBTITLE_SCALE
 495                 ) {
 496
 497                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 498                         i->update (_film, _video_container_size);
 499                 }
 500
 501                 Changed (frequent);
 502
 503         } else if (
 504                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_SCALE ||
 505                 property == VideoContentProperty::VIDEO_FRAME_RATE
 506                 ) {
 507
 508                 Changed (frequent);
 509
 510         } else if (property == ContentProperty::PATH) {
 511
 512                 _have_valid_pieces = false;
 513                 Changed (frequent);
 514         }
 515 }
 516
 517 void
 518 Player::playlist_changed ()
 519 {
 520         _have_valid_pieces = false;
 521         Changed (false);
 522 }
 523
 524 void
 525 Player::set_video_container_size (libdcp::Size s)
 526 {
 527         _video_container_size = s;
 528
 529         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 530         im->make_black ();
 531
 532         _black_frame.reset (
 533                 new PlayerVideoFrame (
 534                         shared_ptr<ImageProxy> (new RawImageProxy (im, _film->log ())),
 535                         Crop(),
 536                         _video_container_size,
 537                         _video_container_size,
 538                         Scaler::from_id ("bicubic"),
 539                         EYES_BOTH,
 540                         PART_WHOLE,
 541                         ColourConversion ()
 542                         )
 543                 );
 544 }
 545
 546 shared_ptr<Resampler>
 547 Player::resampler (shared_ptr<AudioContent> c, bool create)
 548 {
 549         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 550         if (i != _resamplers.end ()) {
 551                 return i->second;
 552         }
 553
 554         if (!create) {
 555                 return shared_ptr<Resampler> ();
 556         }
 557
 558         LOG_GENERAL (
 559                 "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
 560                 );
 561
 562         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 563         _resamplers[c] = r;
 564         return r;
 565 }
 566
 567 void
 568 Player::emit_black ()
 569 {
 570 #ifdef DCPOMATIC_DEBUG
 571         _last_video.reset ();
 572 #endif
 573
 574         Video (_black_frame, _last_emit_was_black, _video_position);
 575         _video_position += _film->video_frames_to_time (1);
 576         _last_emit_was_black = true;
 577 }
 578
 579 void
 580 Player::emit_silence (OutputAudioFrame most)
 581 {
 582         if (most == 0) {
 583                 return;
 584         }
 585
 586         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 587         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 588         silence->make_silent ();
 589         Audio (silence, _audio_position);
 590         _audio_position += _film->audio_frames_to_time (N);
 591 }
 592
 593 void
 594 Player::film_changed (Film::Property p)
 595 {
 596         /* Here we should notice Film properties that affect our output, and
 597            alert listeners that our output now would be different to how it was
 598            last time we were run.
 599         */
 600
 601         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 602                 Changed (false);
 603         }
 604 }
 605
 606 void
 607 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 608 {
 609         if (!image) {
 610                 /* A null image means that we should stop any current subtitles at `from' */
 611                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 612                         i->set_stop (from);
 613                 }
 614         } else {
 615                 _subtitles.push_back (Subtitle (_film, _video_container_size, weak_piece, image, rect, from, to));
 616         }
 617 }
 618
 619 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 620  *  @return false if this could not be done.
 621  */
 622 bool
 623 Player::repeat_last_video ()
 624 {
 625         if (!_last_incoming_video.image || !_have_valid_pieces) {
 626                 return false;
 627         }
 628
 629         process_video (
 630                 _last_incoming_video.weak_piece,
 631                 _last_incoming_video.image,
 632                 _last_incoming_video.eyes,
 633                 _last_incoming_video.part,
 634                 _last_incoming_video.same,
 635                 _last_incoming_video.frame,
 636                 _last_incoming_video.extra
 637                 );
 638
 639         return true;
 640 }