src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "still_image_decoder.h"
  26 #include "still_image_content.h"
  27 #include "moving_image_decoder.h"
  28 #include "moving_image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "playlist.h"
  33 #include "job.h"
  34 #include "image.h"
  35 #include "ratio.h"
  36 #include "resampler.h"
  37 #include "log.h"
  38 #include "scaler.h"
  39
  40 using std::list;
  41 using std::cout;
  42 using std::min;
  43 using std::max;
  44 using std::vector;
  45 using std::pair;
  46 using std::map;
  47 using boost::shared_ptr;
  48 using boost::weak_ptr;
  49 using boost::dynamic_pointer_cast;
  50
  51 class Piece
  52 {
  53 public:
  54         Piece (shared_ptr<Content> c)
  55                 : content (c)
  56                 , video_position (c->position ())
  57                 , audio_position (c->position ())
  58         {}
  59
  60         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  61                 : content (c)
  62                 , decoder (d)
  63                 , video_position (c->position ())
  64                 , audio_position (c->position ())
  65         {}
  66
  67         shared_ptr<Content> content;
  68         shared_ptr<Decoder> decoder;
  69         Time video_position;
  70         Time audio_position;
  71 };
  72
  73 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  74         : _film (f)
  75         , _playlist (p)
  76         , _video (true)
  77         , _audio (true)
  78         , _have_valid_pieces (false)
  79         , _video_position (0)
  80         , _audio_position (0)
  81         , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1))
  82         , _last_emit_was_black (false)
  83 {
  84         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  85         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  86         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  87         set_video_container_size (fit_ratio_within (_film->container()->ratio (), _film->full_frame ()));
  88 }
  89
  90 void
  91 Player::disable_video ()
  92 {
  93         _video = false;
  94 }
  95
  96 void
  97 Player::disable_audio ()
  98 {
  99         _audio = false;
 100 }
 101
 102 bool
 103 Player::pass ()
 104 {
 105         if (!_have_valid_pieces) {
 106                 setup_pieces ();
 107                 _have_valid_pieces = true;
 108         }
 109
 110         Time earliest_t = TIME_MAX;
 111         shared_ptr<Piece> earliest;
 112         enum {
 113                 VIDEO,
 114                 AUDIO
 115         } type = VIDEO;
 116
 117         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 118                 if ((*i)->decoder->done ()) {
 119                         continue;
 120                 }
 121
 122                 if (_video && dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 123                         if ((*i)->video_position < earliest_t) {
 124                                 earliest_t = (*i)->video_position;
 125                                 earliest = *i;
 126                                 type = VIDEO;
 127                         }
 128                 }
 129
 130                 if (_audio && dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 131                         if ((*i)->audio_position < earliest_t) {
 132                                 earliest_t = (*i)->audio_position;
 133                                 earliest = *i;
 134                                 type = AUDIO;
 135                         }
 136                 }
 137         }
 138
 139         if (!earliest) {
 140                 flush ();
 141                 return true;
 142         }
 143
 144         switch (type) {
 145         case VIDEO:
 146                 if (earliest_t > _video_position) {
 147                         emit_black ();
 148                 } else {
 149                         earliest->decoder->pass ();
 150                 }
 151                 break;
 152
 153         case AUDIO:
 154                 if (earliest_t > _audio_position) {
 155                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 156                 } else {
 157                         earliest->decoder->pass ();
 158
 159                         if (earliest->decoder->done()) {
 160                                 shared_ptr<AudioContent> ac = dynamic_pointer_cast<AudioContent> (earliest->content);
 161                                 assert (ac);
 162                                 shared_ptr<Resampler> re = resampler (ac, false);
 163                                 if (re) {
 164                                         shared_ptr<const AudioBuffers> b = re->flush ();
 165                                         if (b->frames ()) {
 166                                                 process_audio (earliest, b, ac->audio_length ());
 167                                         }
 168                                 }
 169                         }
 170                 }
 171                 break;
 172         }
 173
 174         if (_audio) {
 175                 Time audio_done_up_to = TIME_MAX;
 176                 for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 177                         if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 178                                 audio_done_up_to = min (audio_done_up_to, (*i)->audio_position);
 179                         }
 180                 }
 181
 182                 TimedAudioBuffers<Time> tb = _audio_merger.pull (audio_done_up_to);
 183                 Audio (tb.audio, tb.time);
 184                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 185         }
 186
 187         return false;
 188 }
 189
 190 void
 191 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, Eyes eyes, bool same, VideoContent::Frame frame)
 192 {
 193         /* Keep a note of what came in so that we can repeat it if required */
 194         _last_process_video.weak_piece = weak_piece;
 195         _last_process_video.image = image;
 196         _last_process_video.eyes = eyes;
 197         _last_process_video.same = same;
 198         _last_process_video.frame = frame;
 199
 200         shared_ptr<Piece> piece = weak_piece.lock ();
 201         if (!piece) {
 202                 return;
 203         }
 204
 205         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 206         assert (content);
 207
 208         FrameRateConversion frc (content->video_frame_rate(), _film->video_frame_rate());
 209         if (frc.skip && (frame % 2) == 1) {
 210                 return;
 211         }
 212
 213         Time const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate());
 214         if (content->trimmed (relative_time)) {
 215                 return;
 216         }
 217
 218         /* Convert to RGB first, as FFmpeg doesn't seem to like handling YUV images with odd widths */
 219         shared_ptr<Image> work_image = image->scale (image->size (), _film->scaler(), PIX_FMT_RGB24, true);
 220
 221         work_image = work_image->crop (content->crop(), true);
 222
 223         float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio();
 224         libdcp::Size image_size = fit_ratio_within (ratio, _video_container_size);
 225
 226         work_image = work_image->scale (image_size, _film->scaler(), PIX_FMT_RGB24, true);
 227
 228         Time time = content->position() + relative_time - content->trim_start ();
 229
 230         if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) {
 231                 work_image->alpha_blend (_out_subtitle.image, _out_subtitle.position);
 232         }
 233
 234         if (image_size != _video_container_size) {
 235                 assert (image_size.width <= _video_container_size.width);
 236                 assert (image_size.height <= _video_container_size.height);
 237                 shared_ptr<Image> im (new Image (PIX_FMT_RGB24, _video_container_size, true));
 238                 im->make_black ();
 239                 im->copy (work_image, Position<int> ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 240                 work_image = im;
 241         }
 242
 243 #ifdef DCPOMATIC_DEBUG
 244         _last_video = piece->content;
 245 #endif
 246
 247         Video (work_image, eyes, content->colour_conversion(), same, time);
 248         time += TIME_HZ / _film->video_frame_rate();
 249
 250         _last_emit_was_black = false;
 251
 252         _video_position = piece->video_position = time;
 253 }
 254
 255 void
 256 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 257 {
 258         shared_ptr<Piece> piece = weak_piece.lock ();
 259         if (!piece) {
 260                 return;
 261         }
 262
 263         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 264         assert (content);
 265
 266         /* Gain */
 267         if (content->audio_gain() != 0) {
 268                 shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
 269                 gain->apply_gain (content->audio_gain ());
 270                 audio = gain;
 271         }
 272
 273         /* Resample */
 274         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 275                 shared_ptr<Resampler> r = resampler (content, true);
 276                 pair<shared_ptr<const AudioBuffers>, AudioContent::Frame> ro = r->run (audio, frame);
 277                 audio = ro.first;
 278                 frame = ro.second;
 279         }
 280
 281         Time const relative_time = _film->audio_frames_to_time (frame);
 282
 283         if (content->trimmed (relative_time)) {
 284                 return;
 285         }
 286
 287         Time time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time;
 288
 289         /* Remap channels */
 290         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
 291         dcp_mapped->make_silent ();
 292         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 293         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 294                 if (i->first < audio->channels() && i->second < dcp_mapped->channels()) {
 295                         dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 296                 }
 297         }
 298
 299         audio = dcp_mapped;
 300
 301         /* We must cut off anything that comes before the start of all time */
 302         if (time < 0) {
 303                 int const frames = - time * _film->audio_frame_rate() / TIME_HZ;
 304                 if (frames >= audio->frames ()) {
 305                         return;
 306                 }
 307
 308                 shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames));
 309                 trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0);
 310
 311                 audio = trimmed;
 312                 time = 0;
 313         }
 314
 315         _audio_merger.push (audio, time);
 316         piece->audio_position += _film->audio_frames_to_time (audio->frames ());
 317 }
 318
 319 void
 320 Player::flush ()
 321 {
 322         TimedAudioBuffers<Time> tb = _audio_merger.flush ();
 323         if (tb.audio) {
 324                 Audio (tb.audio, tb.time);
 325                 _audio_position += _film->audio_frames_to_time (tb.audio->frames ());
 326         }
 327
 328         while (_video_position < _audio_position) {
 329                 emit_black ();
 330         }
 331
 332         while (_audio_position < _video_position) {
 333                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 334         }
 335
 336 }
 337
 338 /** Seek so that the next pass() will yield (approximately) the requested frame.
 339  *  Pass accurate = true to try harder to get close to the request.
 340  *  @return true on error
 341  */
 342 void
 343 Player::seek (Time t, bool accurate)
 344 {
 345         if (!_have_valid_pieces) {
 346                 setup_pieces ();
 347                 _have_valid_pieces = true;
 348         }
 349
 350         if (_pieces.empty ()) {
 351                 return;
 352         }
 353
 354         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 355                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 356                 if (!vc) {
 357                         continue;
 358                 }
 359
 360                 Time s = t - vc->position ();
 361                 s = max (static_cast<Time> (0), s);
 362                 s = min (vc->length_after_trim(), s);
 363
 364                 (*i)->video_position = (*i)->audio_position = vc->position() + s;
 365
 366                 FrameRateConversion frc (vc->video_frame_rate(), _film->video_frame_rate());
 367                 /* Here we are converting from time (in the DCP) to a frame number in the content.
 368                    Hence we need to use the DCP's frame rate and the double/skip correction, not
 369                    the source's rate.
 370                 */
 371                 VideoContent::Frame f = (s + vc->trim_start ()) * _film->video_frame_rate() / (frc.factor() * TIME_HZ);
 372                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 373         }
 374
 375         _video_position = _audio_position = t;
 376
 377         /* XXX: don't seek audio because we don't need to... */
 378 }
 379
 380 void
 381 Player::setup_pieces ()
 382 {
 383         list<shared_ptr<Piece> > old_pieces = _pieces;
 384
 385         _pieces.clear ();
 386
 387         ContentList content = _playlist->content ();
 388         sort (content.begin(), content.end(), ContentSorter ());
 389
 390         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 391
 392                 shared_ptr<Piece> piece (new Piece (*i));
 393
 394                 /* XXX: into content? */
 395
 396                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 397                 if (fc) {
 398                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 399
 400                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 401                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 402                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1, _2, _3, _4));
 403
 404                         piece->decoder = fd;
 405                 }
 406
 407                 shared_ptr<const StillImageContent> ic = dynamic_pointer_cast<const StillImageContent> (*i);
 408                 if (ic) {
 409                         shared_ptr<StillImageDecoder> id;
 410
 411                         /* See if we can re-use an old StillImageDecoder */
 412                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 413                                 shared_ptr<StillImageDecoder> imd = dynamic_pointer_cast<StillImageDecoder> ((*j)->decoder);
 414                                 if (imd && imd->content() == ic) {
 415                                         id = imd;
 416                                 }
 417                         }
 418
 419                         if (!id) {
 420                                 id.reset (new StillImageDecoder (_film, ic));
 421                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 422                         }
 423
 424                         piece->decoder = id;
 425                 }
 426
 427                 shared_ptr<const MovingImageContent> mc = dynamic_pointer_cast<const MovingImageContent> (*i);
 428                 if (mc) {
 429                         shared_ptr<MovingImageDecoder> md;
 430
 431                         if (!md) {
 432                                 md.reset (new MovingImageDecoder (_film, mc));
 433                                 md->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3, _4));
 434                         }
 435
 436                         piece->decoder = md;
 437                 }
 438
 439                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 440                 if (sc) {
 441                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 442                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 443
 444                         piece->decoder = sd;
 445                 }
 446
 447                 _pieces.push_back (piece);
 448         }
 449 }
 450
 451 void
 452 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 453 {
 454         shared_ptr<Content> c = w.lock ();
 455         if (!c) {
 456                 return;
 457         }
 458
 459         if (
 460                 property == ContentProperty::POSITION || property == ContentProperty::LENGTH ||
 461                 property == ContentProperty::TRIM_START || property == ContentProperty::TRIM_END
 462                 ) {
 463
 464                 _have_valid_pieces = false;
 465                 Changed (frequent);
 466
 467         } else if (property == SubtitleContentProperty::SUBTITLE_OFFSET || property == SubtitleContentProperty::SUBTITLE_SCALE) {
 468
 469                 update_subtitle ();
 470                 Changed (frequent);
 471
 472         } else if (
 473                 property == VideoContentProperty::VIDEO_FRAME_TYPE || property == VideoContentProperty::VIDEO_CROP ||
 474                 property == VideoContentProperty::VIDEO_RATIO
 475                 ) {
 476
 477                 Changed (frequent);
 478         }
 479 }
 480
 481 void
 482 Player::playlist_changed ()
 483 {
 484         _have_valid_pieces = false;
 485         Changed (false);
 486 }
 487
 488 void
 489 Player::set_video_container_size (libdcp::Size s)
 490 {
 491         _video_container_size = s;
 492         _black_frame.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 493         _black_frame->make_black ();
 494 }
 495
 496 shared_ptr<Resampler>
 497 Player::resampler (shared_ptr<AudioContent> c, bool create)
 498 {
 499         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 500         if (i != _resamplers.end ()) {
 501                 return i->second;
 502         }
 503
 504         if (!create) {
 505                 return shared_ptr<Resampler> ();
 506         }
 507
 508         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 509         _resamplers[c] = r;
 510         return r;
 511 }
 512
 513 void
 514 Player::emit_black ()
 515 {
 516 #ifdef DCPOMATIC_DEBUG
 517         _last_video.reset ();
 518 #endif
 519
 520         Video (_black_frame, EYES_BOTH, ColourConversion(), _last_emit_was_black, _video_position);
 521         _video_position += _film->video_frames_to_time (1);
 522         _last_emit_was_black = true;
 523 }
 524
 525 void
 526 Player::emit_silence (OutputAudioFrame most)
 527 {
 528         if (most == 0) {
 529                 return;
 530         }
 531
 532         OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2);
 533         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), N));
 534         silence->make_silent ();
 535         Audio (silence, _audio_position);
 536         _audio_position += _film->audio_frames_to_time (N);
 537 }
 538
 539 void
 540 Player::film_changed (Film::Property p)
 541 {
 542         /* Here we should notice Film properties that affect our output, and
 543            alert listeners that our output now would be different to how it was
 544            last time we were run.
 545         */
 546
 547         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 548                 Changed (false);
 549         }
 550 }
 551
 552 void
 553 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<Image> image, dcpomatic::Rect<double> rect, Time from, Time to)
 554 {
 555         _in_subtitle.piece = weak_piece;
 556         _in_subtitle.image = image;
 557         _in_subtitle.rect = rect;
 558         _in_subtitle.from = from;
 559         _in_subtitle.to = to;
 560
 561         update_subtitle ();
 562 }
 563
 564 void
 565 Player::update_subtitle ()
 566 {
 567         shared_ptr<Piece> piece = _in_subtitle.piece.lock ();
 568         if (!piece) {
 569                 return;
 570         }
 571
 572         if (!_in_subtitle.image) {
 573                 _out_subtitle.image.reset ();
 574                 return;
 575         }
 576
 577         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 578         assert (sc);
 579
 580         dcpomatic::Rect<double> in_rect = _in_subtitle.rect;
 581         libdcp::Size scaled_size;
 582
 583         in_rect.y += sc->subtitle_offset ();
 584
 585         /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 586         scaled_size.width = in_rect.width * _video_container_size.width * sc->subtitle_scale ();
 587         scaled_size.height = in_rect.height * _video_container_size.height * sc->subtitle_scale ();
 588
 589         /* Then we need a corrective translation, consisting of two parts:
 590          *
 591          * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 592          *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 593          *
 594          * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 595          *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 596          *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 597          *
 598          * Combining these two translations gives these expressions.
 599          */
 600
 601         _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2)));
 602         _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2)));
 603
 604         _out_subtitle.image = _in_subtitle.image->scale (
 605                 scaled_size,
 606                 Scaler::from_id ("bicubic"),
 607                 _in_subtitle.image->pixel_format (),
 608                 true
 609                 );
 610         _out_subtitle.from = _in_subtitle.from + piece->content->position ();
 611         _out_subtitle.to = _in_subtitle.to + piece->content->position ();
 612 }
 613
 614 /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles.
 615  *  @return false if this could not be done.
 616  */
 617 bool
 618 Player::repeat_last_video ()
 619 {
 620         if (!_last_process_video.image) {
 621                 return false;
 622         }
 623
 624         process_video (
 625                 _last_process_video.weak_piece,
 626                 _last_process_video.image,
 627                 _last_process_video.eyes,
 628                 _last_process_video.same,
 629                 _last_process_video.frame
 630                 );
 631
 632         return true;
 633 }