src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "image_decoder.h"
  26 #include "image_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "image_proxy.h"
  34 #include "ratio.h"
  35 #include "resampler.h"
  36 #include "log.h"
  37 #include "scaler.h"
  38 #include "player_video_frame.h"
  39 #include "frame_rate_change.h"
  40
  41 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
  42
  43 using std::list;
  44 using std::cout;
  45 using std::min;
  46 using std::max;
  47 using std::vector;
  48 using std::pair;
  49 using std::map;
  50 using boost::shared_ptr;
  51 using boost::weak_ptr;
  52 using boost::dynamic_pointer_cast;
  53
  54 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  55         : _film (f)
  56         , _playlist (p)
  57         , _video (true)
  58         , _audio (true)
  59         , _have_valid_pieces (false)
  60         , _video_position (0)
  61         , _audio_position (0)
  62         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  63         , _last_emit_was_black (false)
  64 {
  65         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  66         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  67         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  68         set_video_container_size (_film->frame_size ());
  69 }
  70
  71 void
  72 Player::disable_video ()
  73 {
  74         _video = false;
  75 }
  76
  77 void
  78 Player::disable_audio ()
  79 {
  80         _audio = false;
  81 }
  82
  83 bool
  84 Player::pass ()
  85 {
  86         if (!_have_valid_pieces) {
  87                 setup_pieces ();
  88         }
  89
  90         Time earliest_t = TIME_MAX;
  91         shared_ptr<Piece> earliest;
  92         enum {
  93                 VIDEO,
  94                 AUDIO
  95         } type = VIDEO;
  96
  97         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
  98                 if ((*i)->decoder->done ()) {
  99                         continue;
 100                 }
 101
 102                 shared_ptr<VideoDecoder> vd = dynamic_pointer_cast<VideoDecoder> ((*i)->decoder);
 103                 shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 104
 105                 if (_video && vd) {
 106                         if ((*i)->video_position < earliest_t) {
 107                                 earliest_t = (*i)->video_position;
 108                                 earliest = *i;
 109                                 type = VIDEO;
 110                         }
 111                 }
 112
 113                 if (_audio && ad && ad->has_audio ()) {
 114                         if ((*i)->audio_position < earliest_t) {
 115                                 earliest_t = (*i)->audio_position;
 116                                 earliest = *i;
 117                                 type = AUDIO;
 118                         }
 119                 }
 120         }
 121
 122         if (!earliest) {
 123                 flush ();
 124                 return true;
 125         }
 126
 127         switch (type) {
 128         case VIDEO:
 129                 if (earliest_t > _video_position) {
 130                         emit_black ();
 131                 } else {
 132                         if (earliest->repeating ()) {
 133                                 earliest->repeat (this);
 134                         } else {
 135                                 earliest->decoder->pass ();
 136                         }
 137                 }
 138                 break;
 139
 140         case AUDIO:
 141                 if (earliest_t > _audio_position) {
 142                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 143                 } else {
 144                         earliest->decoder->pass ();
 145
 146                         if (earliest->decoder->done()) {
 147                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 148                                 assert (ac);
 149                                 shared_ptr<Resampler> re = resampler (ac, false);
 150                                 if (re) {
 151                                         shared_ptr<const AudioBuffers> b = re->flush ();
 152                                         if (b->frames ()) {
 153                                                 process_audio (earliest, b, ac->audio_length (), true);
 154                                         }
 155                                 }
 156                         }
 157                 }
 158                 break;
 159         }
 160
 161         if (_audio) {
 162                 boost::optional<Time> audio_done_up_to;
 163                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 164                         if ((*i)->decoder->done ()) {
 165                                 continue;
 166                         }
 167
 168                         shared_ptr<AudioDecoder> ad = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 169                         if (ad && ad->has_audio ()) {
 170                                 audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position);
 171                         }
 172                 }
 173
 174                 if (audio_done_up_to) {
 175                         TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to.get ());
 176                         Audio (tb.audio, tb.time);
 177                         _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 178                 }
 179         }
 180
 181         return false;
 182 }
 183
 184 /** @param extra Amount of extra time to add to the content frame's time (for repeat) */
 185 void
 186 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const ImageProxy> image, Eyes eyes, Part part, bool same, VideoContent::Frame frame, Time extra)
 187 {
 188         /* Keep a note of what came in so that we can repeat it if required */
 189         _last_incoming_video.weak_piece = weak_piece;
 190         _last_incoming_video.image = image;
 191         _last_incoming_video.eyes = eyes;
 192         _last_incoming_video.part = part;
 193         _last_incoming_video.same = same;
 194         _last_incoming_video.frame = frame;
 195         _last_incoming_video.extra = extra;
 196
 197         shared_ptr<Piece> piece = weak_piece.lock ();
 198         if (!piece) {
 199                 return;
 200         }
 201
 202         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 203         assert (content);
 204
 205         FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate());
 206         if (frc.skip && (frame % 2) == 1) {
 207                 return;
 208         }
 209
 210         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 211         if (content->trimmed (relative_time)) {
 212                 return;
 213         }
 214
 215         Time const time = content->position() + relative_time + extra - content->trim_start ();
 216         libdcp::Size const image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 217
 218         shared_ptr<PlayerVideoFrame> pi (
 219                 new PlayerVideoFrame (
 220                         image,
 221                         content->crop(),
 222                         image_size,
 223                         _video_container_size,
 224                         _film->scaler(),
 225                         eyes,
 226                         part,
 227                         content->colour_conversion()
 228                         )
 229                 );
 230
 231         if (_film->with_subtitles ()) {
 232                 for (list<Subtitle>::const_iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 233                         if (i->covers (time)) {
 234                                 /* This may be true for more than one of _subtitles, but the last (latest-starting)
 235                                    one is the one we want to use, so that's ok.
 236                                 */
 237                                 Position<int> const container_offset (
 238                                         (_video_container_size.width - image_size.width) / 2,
 239                                         (_video_container_size.height - image_size.width) / 2
 240                                         );
 241
 242                                 pi->set_subtitle (i->out_image(), i->out_position() + container_offset);
 243                         }
 244                 }
 245         }
 246
 247         /* Clear out old subtitles */
 248         for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ) {
 249                 list<Subtitle>::iterator j = i;
 250                 ++j;
 251
 252                 if (i->ends_before (time)) {
 253                         _subtitles.erase (i);
 254                 }
 255
 256                 i = j;
 257         }
 258
 259 #ifdef DCPOMATIC_DEBUG
 260         _last_video = piece->content;
 261 #endif
 262
 263         Video (pi, same, time);
 264
 265         _last_emit_was_black = false;
 266         _video_position = piece->video_position = (time + TIME_HZ / _film->video_frame_rate());
 267
 268         if (frc.repeat > 1 && !piece->repeating ()) {
 269                 piece->set_repeat (_last_incoming_video, frc.repeat - 1);
 270         }
 271 }
 272
 273 /** @param already_resampled true if this data has already been through the chain up to the resampler */
 274 void
 275 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame, bool already_resampled)
 276 {
 277         shared_ptr<Piece> piece = weak_piece.lock ();
 278         if (!piece) {
 279                 return;
 280         }
 281
 282         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 283         assert (content);
 284
 285         if (!already_resampled) {
 286                 /* Gain */
 287                 if (content->audio_gain() != 0) {
 288                         shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 289                         gain->apply_gain (content->audio_gain ());
 290                         audio = gain;
 291                 }
 292
 293                 /* Resample */
 294                 if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 295                         shared_ptr<Resampler> r = resampler (content, true);
 296                         pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 297                         audio = ro.first;
 298                         frame = ro.second;
 299                 }
 300         }
 301
 302         Time const relative_time = _film->audio_frames_to_time (frame);
 303
 304         if (content->trimmed (relative_time)) {
 305                 return;
 306         }
 307
 308         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start ();
 309
 310         /* Remap channels */
 311         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 312         dcp_mapped->make_silent ();
 313
 314         AudioMapping map = content->audio_mapping ();
 315         for (int i = 0; i < map.content_channels(); ++i) {
 316                 for (int j = 0; j < _film->audio_channels(); ++j) {
 317                         if (map.get (i, static_cast<libdcp::Channel> (j)) > 0) {
 318                                 dcp_mapped->accumulate_channel (
 319                                         audio.get(),
 320                                         i,
 321                                         static_cast<libdcp::Channel> (j),
 322                                         map.get (i, static_cast<libdcp::Channel> (j))
 323                                         );
 324                         }
 325                 }
 326         }
 327
 328         audio = dcp_mapped;
 329
 330         /* We must cut off anything that comes before the start of all time */
 331         if (time < 0) {
 332                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 333                 if (frames >= audio->frames ()) {
 334                         return;
 335                 }
 336
 337                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 338                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 339
 340                 audio = trimmed;
 341                 time = 0;
 342         }
 343
 344         _audio_merger.push (audio, time);
 345         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 346 }
 347
 348 void
 349 Player::flush ()
 350 {
 351         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 352         if (_audio && tb.audio) {
 353                 Audio (tb.audio, tb.time);
 354                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 355         }
 356
 357         while (_video && _video_position < _audio_position) {
 358                 emit_black ();
 359         }
 360
 361         while (_audio && _audio_position < _video_position) {
 362                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 363         }
 364
 365 }
 366
 367 /** Seek so that the next pass() will yield (approximately) the requested frame.
 368  *  Pass accurate = true to try harder to get close to the request.
 369  *  @return true on error
 370  */
 371 void
 372 Player::seek (Time t, bool accurate)
 373 {
 374         if (!_have_valid_pieces) {
 375                 setup_pieces ();
 376         }
 377
 378         if (_pieces.empty ()) {
 379                 return;
 380         }
 381
 382         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 383                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 384                 if (!vc) {
 385                         continue;
 386                 }
 387
 388                 /* s is the offset of t from the start position of this content */
 389                 Time s = t - vc->position ();
 390                 s = max (static_cast<Time> (0), s);
 391                 s = min (vc->length_after_trim(), s);
 392
 393                 /* Hence set the piece positions to the `global' time */
 394                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 395
 396                 /* And seek the decoder */
 397                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (
 398                         vc->time_to_content_video_frames (s + vc->trim_start ()), accurate
 399                         );
 400
 401                 (*i)->reset_repeat ();
 402         }
 403
 404         _video_position = _audio_position = t;
 405
 406         /* XXX: don't seek audio because we don't need to... */
 407 }
 408
 409 void
 410 Player::setup_pieces ()
 411 {
 412         list<shared_ptr<Piece> > old_pieces = _pieces;
 413
 414         _pieces.clear ();
 415
 416         ContentList content = _playlist->content ();
 417         sort (content.begin(), content.end(), ContentSorter ());
 418
 419         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 420
 421                 if (!(*i)->paths_valid ()) {
 422                         continue;
 423                 }
 424
 425                 shared_ptr<Piece> piece (new Piece (*i));
 426
 427                 /* XXX: into content? */
 428
 429                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 430                 if (fc) {
 431                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 432
 433                         fd->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, _5, 0));
 434                         fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2, false));
 435                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr<Piece> (piece), _1, _2, _3, _4));
 436
 437                         fd->seek (fc->time_to_content_video_frames (fc->trim_start ()), true);
 438                         piece->decoder = fd;
 439                 }
 440
 441                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 442                 if (ic) {
 443                         bool reusing = false;
 444
 445                         /* See if we can re-use an old ImageDecoder */
 446                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 447                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 448                                 if (imd && imd->content() == ic) {
 449                                         piece = *j;
 450                                         reusing = true;
 451                                 }
 452                         }
 453
 454                         if (!reusing) {
 455                                 shared_ptr<ImageDecoder> id (new ImageDecoder (_film, ic));
 456                                 id->Video.connect (bind (&Player::process_video, this, weak_ptr<Piece> (piece), _1, _2, _3, _4, _5, 0));
 457                                 piece->decoder = id;
 458                         }
 459                 }
 460
 461                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 462                 if (sc) {
 463                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 464                         sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr<Piece> (piece), _1, _2, false));
 465
 466                         piece->decoder = sd;
 467                 }
 468
 469                 _pieces.push_back (piece);
 470         }
 471
 472         _have_valid_pieces = true;
 473 }
 474
 475 void
 476 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 477 {
 478         shared_ptr<Content> c = w.lock ();
 479         if (!c) {
 480                 return;
 481         }
 482
 483         if (
 484                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 485                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END ||
 486                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 487                 ) {
 488
 489                 _have_valid_pieces = false;
 490                 Changed (frequent);
 491
 492         } else if (
 493                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 494                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 495                 property == SubtitleContentProperty::SUBTITLE_SCALE
 496                 ) {
 497
 498                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 499                         i->update (_film, _video_container_size);
 500                 }
 501
 502                 Changed (frequent);
 503
 504         } else if (
 505                 property == VideoContentProperty::VIDEO_CROP || property == VideoContentProperty::VIDEO_SCALE ||
 506                 property == VideoContentProperty::VIDEO_FRAME_RATE
 507                 ) {
 508
 509                 Changed (frequent);
 510
 511         } else if (property == ContentProperty::PATH) {
 512
 513                 _have_valid_pieces = false;
 514                 Changed (frequent);
 515         }
 516 }
 517
 518 void
 519 Player::playlist_changed ()
 520 {
 521         _have_valid_pieces = false;
 522         Changed (false);
 523 }
 524
 525 void
 526 Player::set_video_container_size (libdcp::Size s)
 527 {
 528         _video_container_size = s;
 529
 530         shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 531         im->make_black ();
 532
 533         _black_frame.reset (
 534                 new PlayerVideoFrame (
 535                         shared_ptr<ImageProxy> (new RawImageProxy (im, _film->log ())),
 536                         Crop(),
 537                         _video_container_size,
 538                         _video_container_size,
 539                         Scaler::from_id ("bicubic"),
 540                         EYES_BOTH,
 541                         PART_WHOLE,
 542                         ColourConversion ()
 543                         )
 544                 );
 545 }
 546
 547 shared_ptr<Resampler>
 548 Player::resampler (shared_ptr<AudioContent> c, bool create)
 549 {
 550         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 551         if (i != _resamplers.end ()) {
 552                 return i->second;
 553         }
 554
 555         if (!create) {
 556                 return shared_ptr<Resampler> ();
 557         }
 558
 559         LOG_GENERAL (
 560                 "Creating new resampler for %1 to %2 with %3 channels", c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()
 561                 );
 562
 563         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 564         _resamplers[c] = r;
 565         return r;
 566 }
 567
 568 void
 569 Player::emit_black ()
 570 {
 571 #ifdef DCPOMATIC_DEBUG
 572         _last_video.reset ();
 573 #endif
 574
 575         Video (_black_frame, _last_emit_was_black, _video_position);
 576         _video_position += _film->video_frames_to_time (1);
 577         _last_emit_was_black = true;
 578 }
 579
 580 void
 581 Player::emit_silence (OutputAudioFrame most)
 582 {
 583         if (most == 0) {
 584                 return;
 585         }
 586
 587         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 588         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 589         silence->make_silent ();
 590         Audio (silence, _audio_position);
 591         _audio_position += _film->audio_frames_to_time (N);
 592 }
 593
 594 void
 595 Player::film_changed (Film::Property p)
 596 {
 597         /* Here we should notice Film properties that affect our output, and
 598            alert listeners that our output now would be different to how it was
 599            last time we were run.
 600         */
 601
 602         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 603                 Changed (false);
 604         }
 605 }
 606
 607 void
 608 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 609 {
 610         if (!image) {
 611                 /* A null image means that we should stop any current subtitles at `from' */
 612                 for (list<Subtitle>::iterator i = _subtitles.begin(); i != _subtitles.end(); ++i) {
 613                         i->set_stop (from);
 614                 }
 615         } else {
 616                 _subtitles.push_back (Subtitle (_film, _video_container_size, weak_piece, image, rect, from, to));
 617         }
 618 }
 619
 620 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 621  *  @return false if this could not be done.
 622  */
 623 bool
 624 Player::repeat_last_video ()
 625 {
 626         if (!_last_incoming_video.image || !_have_valid_pieces) {
 627                 return false;
 628         }
 629
 630         process_video (
 631                 _last_incoming_video.weak_piece,
 632                 _last_incoming_video.image,
 633                 _last_incoming_video.eyes,
 634                 _last_incoming_video.part,
 635                 _last_incoming_video.same,
 636                 _last_incoming_video.frame,
 637                 _last_incoming_video.extra
 638                 );
 639
 640         return true;
 641 }