src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "image_proxy.h"
  34 #include "ratio.h"
  35 #include "resampler.h"
  36 #include "log.h"
  37 #include "scaler.h"
  38 #include "player_video_frame.h"
  39
  40 using std::list;
  41 using std::cout;
  42 using std::min;
  43 using std::max;
  44 using std::vector;
  45 using std::pair;
  46 using std::map;
  47 using boost::shared_ptr;
  48 using boost::weak_ptr;
  49 using boost::dynamic_pointer_cast;
  50
  51 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  52         : _film (f)
  53         , _playlist (p)
  54         , _video (true)
  55         , _audio (true)
  56         , _have_valid_pieces (false)
  57         , _video_position (0)
  58         , _audio_position (0)
  59         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  60         , _last_emit_was_black (false)
  61 {
  62         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  63         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  64         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  65         set_video_container_size (_film->frame_size ());
  66 }
  67
  68 void
  69 Player::disable_video ()
  70 {
  71         _video = false;
  72 }
  73
  74 void
  75 Player::disable_audio ()
  76 {
  77         _audio = false;
  78 }
  79
  80 bool
  81 Player::pass ()
  82 {
  83         if (!_have_valid_pieces) {
  84                 setup_pieces ();
  85         }
  86
  87         Time earliest_t = TIME_MAX;
  88         shared_ptr<Piece> earliest;
  89         enum {
  90                 VIDEO,
  91                 AUDIO
  92         } type = VIDEO;
  93
  94         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
  95                 if ((*i)->decoder->done ()) {
  96                         continue;
  97                 }
  98
  99                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
 100                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 101
 102                 if (_video && vd) {
 103                         if ((*i)->video_position < earliest_t) {
 104                                 earliest_t = (*i)->video_position;
 105                                 earliest = *i;
 106                                 type = VIDEO;
 107                         }
 108                 }
 109
 110                 if (_audio && ad && ad->has_audio ()) {
 111                         if ((*i)->audio_position < earliest_t) {
 112                                 earliest_t = (*i)->audio_position;
 113                                 earliest = *i;
 114                                 type = AUDIO;
 115                         }
 116                 }
 117         }
 118
 119         if (!earliest) {
 120                 flush ();
 121                 return true;
 122         }
 123
 124         switch (type) {
 125         case VIDEO:
 126                 if (earliest_t > _video_position) {
 127                         emit_black ();
 128                 } else {
 129                         if (earliest->repeating ()) {
 130                                 earliest->repeat (this);
 131                         } else {
 132                                 earliest->decoder->pass ();
 133                         }
 134                 }
 135                 break;
 136
 137         case AUDIO:
 138                 if (earliest_t > _audio_position) {
 139                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 140                 } else {
 141                         earliest->decoder->pass ();
 142
 143                         if (earliest->decoder->done()) {
 144                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 145                                 assert (ac);
 146                                 shared_ptr<Resampler> re = resampler (ac, false);
 147                                 if (re) {
 148                                         shared_ptr<const AudioBuffers> b = re->flush ();
 149                                         if (b->frames ()) {
 150                                                 process_audio (earliest, b, ac->audio_length ());
 151                                         }
 152                                 }
 153                         }
 154                 }
 155                 break;
 156         }
 157
 158         if (_audio) {
 159                 boost::optional<Time> audio_done_up_to;
 160                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 161                         if ((*i)->decoder->done ()) {
 162                                 continue;
 163                         }
 164
 165                         shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 166                         if (ad && ad->has_audio ()) {
 167                                 audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position);
 168                         }
 169                 }
 170
 171                 if (audio_done_up_to) {
 172                         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to.get ());
 173                         Audio (tb.audio, tb.time);
 174                         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 175                 }
 176         }
 177
 178         return false;
 179 }
 180
 181 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
 182 void
 183 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const ImageProxy> image, Eyes eyes, Part part, bool same, VideoContent::Frame frame, Time extra)
 184 {
 185         /* Keep a note of what came in so that we can repeat it if required */
 186         _last_incoming_video.weak_piece = weak_piece;
 187         _last_incoming_video.image = image;
 188         _last_incoming_video.eyes = eyes;
 189         _last_incoming_video.part = part;
 190         _last_incoming_video.same = same;
 191         _last_incoming_video.frame = frame;
 192         _last_incoming_video.extra = extra;
 193
 194         shared_ptr<Piece> piece = weak_piece.lock ();
 195         if (!piece) {
 196                 return;
 197         }
 198
 199         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 200         assert (content);
 201
 202         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 203         if (frc.skip && (frame % 2) == 1) {
 204                 return;
 205         }
 206
 207         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 208         if (content->trimmed (relative_time)) {
 209                 return;
 210         }
 211
 212         Time const time = content->position() + relative_time + extra - content->trim_start ();
 213         libdcp::Size const image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 214
 215         shared_ptr<PlayerVideoFrame> pi (
 216                 new PlayerVideoFrame (
 217                         image,
 218                         content->crop(),
 219                         image_size,
 220                         _video_container_size,
 221                         _film->scaler(),
 222                         eyes,
 223                         part,
 224                         content->colour_conversion()
 225                         )
 226                 );
 227
 228         if (_film->with_subtitles ()) {
 229                 for (list<Subtitle>::const_iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 230                         if (i->covers (time)) {
 231                                 /* This may be true for more than one of _subtitles, but the last (latest-starting)
 232                                    one is the one we want to use, so that's ok.
 233                                 */
 234                                 Position<int> const container_offset (
 235                                         (_video_container_size.width - image_size.width) / 2,
 236                                         (_video_container_size.height - image_size.width) / 2
 237                                         );
 238
 239                                 pi->set_subtitle (i->out_image(), i->out_position() + container_offset);
 240                         }
 241                 }
 242         }
 243
 244         /* Clear out old subtitles */
 245         for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ) {
 246                 list<Subtitle>::iterator j = i;
 247                 ++j;
 248
 249                 if (i->ends_before (time)) {
 250                         _subtitles.erase (i);
 251                 }
 252
 253                 i = j;
 254         }
 255
 256 #ifdef DCPOMATIC_DEBUG
 257         _last_video = piece->content;
 258 #endif
 259
 260         Video (pi, same, time);
 261
 262         _last_emit_was_black = false;
 263         _video_position = piece->video_position = (time + TIME_HZ / _film->video_frame_rate());
 264
 265         if (frc.repeat > 1 && !piece->repeating ()) {
 266                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
 267         }
 268 }
 269
 270 void
 271 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 272 {
 273         shared_ptr<Piece> piece = weak_piece.lock ();
 274         if (!piece) {
 275                 return;
 276         }
 277
 278         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 279         assert (content);
 280
 281         /* Gain */
 282         if (content->audio_gain() != 0) {
 283                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 284                 gain->apply_gain (content->audio_gain ());
 285                 audio = gain;
 286         }
 287
 288         /* Resample */
 289         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 290                 shared_ptr<Resampler> r = resampler (content, true);
 291                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 292                 audio = ro.first;
 293                 frame = ro.second;
 294         }
 295
 296         Time const relative_time = _film->audio_frames_to_time (frame);
 297
 298         if (content->trimmed (relative_time)) {
 299                 return;
 300         }
 301
 302         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
 303
 304         /* Remap channels */
 305         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 306         dcp_mapped->make_silent ();
 307
 308         AudioMapping map = content->audio_mapping ();
 309         for (int i = 0; i < map.content_channels(); ++i) {
 310                 for (int j = 0; j < _film->audio_channels(); ++j) {
 311                         if (map.get (i, static_cast<libdcp::Channel> (j)) > 0) {
 312                                 dcp_mapped->accumulate_channel (
 313                                         audio.get(),
 314                                         i,
 315                                         static_cast<libdcp::Channel> (j),
 316                                         map.get (i, static_cast<libdcp::Channel> (j))
 317                                         );
 318                         }
 319                 }
 320         }
 321
 322         audio = dcp_mapped;
 323
 324         /* We must cut off anything that comes before the start of all time */
 325         if (time < 0) {
 326                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 327                 if (frames >= audio->frames ()) {
 328                         return;
 329                 }
 330
 331                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 332                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 333
 334                 audio = trimmed;
 335                 time = 0;
 336         }
 337
 338         _audio_merger.push (audio, time);
 339         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 340 }
 341
 342 void
 343 Player::flush ()
 344 {
 345         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 346         if (_audio && tb.audio) {
 347                 Audio (tb.audio, tb.time);
 348                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 349         }
 350
 351         while (_video && _video_position < _audio_position) {
 352                 emit_black ();
 353         }
 354
 355         while (_audio && _audio_position < _video_position) {
 356                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 357         }
 358
 359 }
 360
 361 /** Seek so that the next pass() will yield (approximately) the requested frame.
 362  *  Pass accurate = true to try harder to get close to the request.
 363  *  @return true on error
 364  */
 365 void
 366 Player::seek (Time t, bool accurate)
 367 {
 368         if (!_have_valid_pieces) {
 369                 setup_pieces ();
 370         }
 371
 372         if (_pieces.empty ()) {
 373                 return;
 374         }
 375
 376         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 377                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 378                 if (!vc) {
 379                         continue;
 380                 }
 381
 382                 /* s is the offset of t from the start position of this content */
 383                 Time s = t - vc->position ();
 384                 s = max (static_cast<Time> (0), s);
 385                 s = min (vc->length_after_trim(), s);
 386
 387                 /* Hence set the piece positions to the `global' time */
 388                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 389
 390                 /* And seek the decoder */
 391                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
 392                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
 393                         );
 394
 395                 (*i)->reset_repeat ();
 396         }
 397
 398         _video_position = _audio_position = t;
 399
 400         /* XXX: don't seek audio because we don't need to... */
 401 }
 402
 403 void
 404 Player::setup_pieces ()
 405 {
 406         list<shared_ptr<Piece> > old_pieces = _pieces;
 407
 408         _pieces.clear ();
 409
 410         ContentList content = _playlist->content ();
 411         sort (content.begin(), content.end(), ContentSorter ());
 412
 413         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 414
 415                 if (!(*i)->paths_valid ()) {
 416                         continue;
 417                 }
 418
 419                 shared_ptr<Piece> piece (new Piece (*i));
 420
 421                 /* XXX: into content? */
 422
 423                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 424                 if (fc) {
 425                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 426
 427                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, _5, 0));
 428                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 429                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
 430
 431                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
 432                         piece->decoder = fd;
 433                 }
 434
 435                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 436                 if (ic) {
 437                         bool reusing = false;
 438
 439                         /* See if we can re-use an old ImageDecoder */
 440                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 441                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 442                                 if (imd && imd->content() == ic) {
 443                                         piece = *j;
 444                                         reusing = true;
 445                                 }
 446                         }
 447
 448                         if (!reusing) {
 449                                 shared_ptr<ImageDecoder> id (new ImageDecoder (_film, ic));
 450                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, _5, 0));
 451                                 piece->decoder = id;
 452                         }
 453                 }
 454
 455                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 456                 if (sc) {
 457                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 458                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2));
 459
 460                         piece->decoder = sd;
 461                 }
 462
 463                 _pieces.push_back (piece);
 464         }
 465
 466         _have_valid_pieces = true;
 467 }
 468
 469 void
 470 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 471 {
 472         shared_ptr<Content> c = w.lock ();
 473         if (!c) {
 474                 return;
 475         }
 476
 477         if (
 478                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 479                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 480                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 481                 ) {
 482
 483                 _have_valid_pieces = false;
 484                 Changed (frequent);
 485
 486         } else if (
 487                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 488                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 489                 property == SubtitleContentProperty::SUBTITLE_SCALE
 490                 ) {
 491
 492                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 493                         i->update (_film, _video_container_size);
 494                 }
 495
 496                 Changed (frequent);
 497
 498         } else if (
 499                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_SCALE ||
 500                 property == VideoContentProperty::VIDEO_FRAME_RATE
 501                 ) {
 502
 503                 Changed (frequent);
 504
 505         } else if (property == ContentProperty::PATH) {
 506
 507                 _have_valid_pieces = false;
 508                 Changed (frequent);
 509         }
 510 }
 511
 512 void
 513 Player::playlist_changed ()
 514 {
 515         _have_valid_pieces = false;
 516         Changed (false);
 517 }
 518
 519 void
 520 Player::set_video_container_size (libdcp::Size s)
 521 {
 522         _video_container_size = s;
 523
 524         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 525         im->make_black ();
 526
 527         _black_frame.reset (
 528                 new PlayerVideoFrame (
 529                         shared_ptr<ImageProxy> (new RawImageProxy (im)),
 530                         Crop(),
 531                         _video_container_size,
 532                         _video_container_size,
 533                         Scaler::from_id ("bicubic"),
 534                         EYES_BOTH,
 535                         PART_WHOLE,
 536                         ColourConversion ()
 537                         )
 538                 );
 539 }
 540
 541 shared_ptr<Resampler>
 542 Player::resampler (shared_ptr<AudioContent> c, bool create)
 543 {
 544         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 545         if (i != _resamplers.end ()) {
 546                 return i->second;
 547         }
 548
 549         if (!create) {
 550                 return shared_ptr<Resampler> ();
 551         }
 552
 553         _film->log()->log (
 554                 String::compose (
 555                         "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
 556                         )
 557                 );
 558
 559         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 560         _resamplers[c] = r;
 561         return r;
 562 }
 563
 564 void
 565 Player::emit_black ()
 566 {
 567 #ifdef DCPOMATIC_DEBUG
 568         _last_video.reset ();
 569 #endif
 570
 571         Video (_black_frame, _last_emit_was_black, _video_position);
 572         _video_position += _film->video_frames_to_time (1);
 573         _last_emit_was_black = true;
 574 }
 575
 576 void
 577 Player::emit_silence (OutputAudioFrame most)
 578 {
 579         if (most == 0) {
 580                 return;
 581         }
 582
 583         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 584         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 585         silence->make_silent ();
 586         Audio (silence, _audio_position);
 587         _audio_position += _film->audio_frames_to_time (N);
 588 }
 589
 590 void
 591 Player::film_changed (Film::Property p)
 592 {
 593         /* Here we should notice Film properties that affect our output, and
 594            alert listeners that our output now would be different to how it was
 595            last time we were run.
 596         */
 597
 598         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 599                 Changed (false);
 600         }
 601 }
 602
 603 void
 604 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 605 {
 606         if (!image) {
 607                 /* A null image means that we should stop any current subtitles at `from' */
 608                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 609                         i->set_stop (from);
 610                 }
 611         } else {
 612                 _subtitles.push_back (Subtitle (_film, _video_container_size, weak_piece, image, rect, from, to));
 613         }
 614 }
 615
 616 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 617  *  @return false if this could not be done.
 618  */
 619 bool
 620 Player::repeat_last_video ()
 621 {
 622         if (!_last_incoming_video.image || !_have_valid_pieces) {
 623                 return false;
 624         }
 625
 626         process_video (
 627                 _last_incoming_video.weak_piece,
 628                 _last_incoming_video.image,
 629                 _last_incoming_video.eyes,
 630                 _last_incoming_video.part,
 631                 _last_incoming_video.same,
 632                 _last_incoming_video.frame,
 633                 _last_incoming_video.extra
 634                 );
 635
 636         return true;
 637 }