src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "imagemagick_decoder.h"
  26 #include "imagemagick_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "playlist.h"
  30 #include "job.h"
  31 #include "image.h"
  32 #include "ratio.h"
  33 #include "resampler.h"
  34 #include "subtitle.h"
  35
  36 using std::list;
  37 using std::cout;
  38 using std::min;
  39 using std::max;
  40 using std::vector;
  41 using std::pair;
  42 using std::map;
  43 using boost::shared_ptr;
  44 using boost::weak_ptr;
  45 using boost::dynamic_pointer_cast;
  46
  47 #define DEBUG_PLAYER 1
  48
  49 class Piece
  50 {
  51 public:
  52         Piece (shared_ptr<Content> c)
  53                 : content (c)
  54                 , video_position (c->start ())
  55                 , audio_position (c->start ())
  56         {}
  57
  58         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  59                 : content (c)
  60                 , decoder (d)
  61                 , video_position (c->start ())
  62                 , audio_position (c->start ())
  63         {}
  64
  65         shared_ptr<Content> content;
  66         shared_ptr<Decoder> decoder;
  67         Time video_position;
  68         Time audio_position;
  69 };
  70
  71 #ifdef DEBUG_PLAYER
  72 std::ostream& operator<<(std::ostream& s, Piece const & p)
  73 {
  74         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
  75                 s << "\tffmpeg     ";
  76         } else if (dynamic_pointer_cast<ImageMagickContent> (p.content)) {
  77                 s << "\timagemagick";
  78         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
  79                 s << "\tsndfile    ";
  80         }
  81
  82         s << " at " << p.content->start() << " until " << p.content->end();
  83
  84         return s;
  85 }
  86 #endif
  87
  88 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  89         : _film (f)
  90         , _playlist (p)
  91         , _video (true)
  92         , _audio (true)
  93         , _have_valid_pieces (false)
  94         , _video_position (0)
  95         , _audio_position (0)
  96         , _audio_buffers (f->dcp_audio_channels(), 0)
  97 {
  98         _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  99         _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2));
 100         _film->Changed.connect (bind (&Player::film_changed, this, _1));
 101         set_video_container_size (_film->container()->size (_film->full_frame ()));
 102 }
 103
 104 void
 105 Player::disable_video ()
 106 {
 107         _video = false;
 108 }
 109
 110 void
 111 Player::disable_audio ()
 112 {
 113         _audio = false;
 114 }
 115
 116 bool
 117 Player::pass ()
 118 {
 119         if (!_have_valid_pieces) {
 120                 setup_pieces ();
 121                 _have_valid_pieces = true;
 122         }
 123
 124 #ifdef DEBUG_PLAYER
 125         cout << "= PASS\n";
 126 #endif
 127
 128         Time earliest_t = TIME_MAX;
 129         shared_ptr<Piece> earliest;
 130         enum {
 131                 VIDEO,
 132                 AUDIO
 133         } type = VIDEO;
 134
 135         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 136                 if ((*i)->decoder->done ()) {
 137                         continue;
 138                 }
 139
 140                 if (dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 141                         if ((*i)->video_position < earliest_t) {
 142                                 earliest_t = (*i)->video_position;
 143                                 earliest = *i;
 144                                 type = VIDEO;
 145                         }
 146                 }
 147
 148                 if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 149                         if ((*i)->audio_position < earliest_t) {
 150                                 earliest_t = (*i)->audio_position;
 151                                 earliest = *i;
 152                                 type = AUDIO;
 153                         }
 154                 }
 155         }
 156
 157         if (!earliest) {
 158 #ifdef DEBUG_PLAYER
 159                 cout << "no earliest piece.\n";
 160 #endif
 161
 162                 flush ();
 163                 return true;
 164         }
 165
 166         switch (type) {
 167         case VIDEO:
 168                 if (earliest_t > _video_position) {
 169 #ifdef DEBUG_PLAYER
 170                         cout << "no video here; emitting black frame.\n";
 171 #endif
 172                         emit_black ();
 173                 } else {
 174 #ifdef DEBUG_PLAYER
 175                         cout << "Pass " << *earliest << "\n";
 176 #endif
 177                         earliest->decoder->pass ();
 178                 }
 179                 break;
 180
 181         case AUDIO:
 182                 if (earliest_t > _audio_position) {
 183 #ifdef DEBUG_PLAYER
 184                         cout << "no audio here; emitting silence.\n";
 185 #endif
 186                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 187                 } else {
 188 #ifdef DEBUG_PLAYER
 189                         cout << "Pass " << *earliest << "\n";
 190 #endif
 191                         earliest->decoder->pass ();
 192                 }
 193                 break;
 194         }
 195
 196 #ifdef DEBUG_PLAYER
 197         cout << "\tpost pass " << _video_position << " " << _audio_position << "\n";
 198 #endif
 199
 200         return false;
 201 }
 202
 203 void
 204 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, bool same, VideoContent::Frame frame)
 205 {
 206         shared_ptr<Piece> piece = weak_piece.lock ();
 207         if (!piece) {
 208                 return;
 209         }
 210
 211         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 212         assert (content);
 213
 214         FrameRateConversion frc (content->video_frame_rate(), _film->dcp_video_frame_rate());
 215         if (frc.skip && (frame % 2) == 1) {
 216                 return;
 217         }
 218
 219         shared_ptr<Image> work_image = image->crop (content->crop(), true);
 220
 221         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
 222
 223         work_image = work_image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
 224
 225         Time time = content->start() + (frame * frc.factor() * TIME_HZ / _film->dcp_video_frame_rate());
 226
 227         if (_film->with_subtitles ()) {
 228                 shared_ptr<Subtitle> sub;
 229                 if (_subtitle && _subtitle->displayed_at (time - _subtitle_offset)) {
 230                         sub = _subtitle->subtitle ();
 231                 }
 232
 233                 if (sub) {
 234                         dcpomatic::Rect const tx = subtitle_transformed_area (
 235                                 float (image_size.width) / content->video_size().width,
 236                                 float (image_size.height) / content->video_size().height,
 237                                 sub->area(), _film->subtitle_offset(), _film->subtitle_scale()
 238                                 );
 239
 240                         shared_ptr<Image> im = sub->image()->scale (tx.size(), _film->scaler(), true);
 241                         work_image->alpha_blend (im, tx.position());
 242                 }
 243         }
 244
 245         if (image_size != _video_container_size) {
 246                 assert (image_size.width <= _video_container_size.width);
 247                 assert (image_size.height <= _video_container_size.height);
 248                 shared_ptr<Image> im (new SimpleImage (PIX_FMT_RGB24, _video_container_size, true));
 249                 im->make_black ();
 250                 im->copy (work_image, Position ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 251                 work_image = im;
 252         }
 253
 254         Video (work_image, same, time);
 255         time += TIME_HZ / _film->dcp_video_frame_rate();
 256
 257         if (frc.repeat) {
 258                 Video (work_image, true, time);
 259                 time += TIME_HZ / _film->dcp_video_frame_rate();
 260         }
 261
 262         _video_position = piece->video_position = time;
 263 }
 264
 265 void
 266 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 267 {
 268         shared_ptr<Piece> piece = weak_piece.lock ();
 269         if (!piece) {
 270                 return;
 271         }
 272
 273         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 274         assert (content);
 275
 276         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 277                 audio = resampler(content)->run (audio);
 278         }
 279
 280         /* Remap channels */
 281         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->dcp_audio_channels(), audio->frames()));
 282         dcp_mapped->make_silent ();
 283         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 284         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 285                 dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 286         }
 287
 288         audio = dcp_mapped;
 289
 290         /* The time of this audio may indicate that some of our buffered audio is not going to
 291            be added to any more, so it can be emitted.
 292         */
 293
 294         Time const time = content->start() + (frame * TIME_HZ / _film->dcp_audio_frame_rate());
 295
 296         if (time > _audio_position) {
 297                 /* We can emit some audio from our buffers */
 298                 OutputAudioFrame const N = _film->time_to_audio_frames (time - _audio_position);
 299                 assert (N <= _audio_buffers.frames());
 300                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), N));
 301                 emit->copy_from (&_audio_buffers, N, 0, 0);
 302                 Audio (emit, _audio_position);
 303                 _audio_position = piece->audio_position = time + _film->audio_frames_to_time (N);
 304
 305                 /* And remove it from our buffers */
 306                 if (_audio_buffers.frames() > N) {
 307                         _audio_buffers.move (N, 0, _audio_buffers.frames() - N);
 308                 }
 309                 _audio_buffers.set_frames (_audio_buffers.frames() - N);
 310         }
 311
 312         /* Now accumulate the new audio into our buffers */
 313         _audio_buffers.ensure_size (_audio_buffers.frames() + audio->frames());
 314         _audio_buffers.accumulate_frames (audio.get(), 0, 0, audio->frames ());
 315         _audio_buffers.set_frames (_audio_buffers.frames() + audio->frames());
 316 }
 317
 318 void
 319 Player::flush ()
 320 {
 321         if (_audio_buffers.frames() > 0) {
 322                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), _audio_buffers.frames()));
 323                 emit->copy_from (&_audio_buffers, _audio_buffers.frames(), 0, 0);
 324                 Audio (emit, _audio_position);
 325                 _audio_position += _film->audio_frames_to_time (_audio_buffers.frames ());
 326                 _audio_buffers.set_frames (0);
 327         }
 328
 329         while (_video_position < _audio_position) {
 330                 emit_black ();
 331         }
 332
 333         while (_audio_position < _video_position) {
 334                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 335         }
 336
 337 }
 338
 339 /** @return true on error */
 340 void
 341 Player::seek (Time t, bool accurate)
 342 {
 343         if (!_have_valid_pieces) {
 344                 setup_pieces ();
 345                 _have_valid_pieces = true;
 346         }
 347
 348         if (_pieces.empty ()) {
 349                 return;
 350         }
 351
 352         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 353                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 354                 if (!vc) {
 355                         continue;
 356                 }
 357
 358                 Time s = t - vc->start ();
 359                 s = max (static_cast<Time> (0), s);
 360                 s = min (vc->length(), s);
 361
 362                 FrameRateConversion frc (vc->video_frame_rate(), _film->dcp_video_frame_rate());
 363                 VideoContent::Frame f = s * vc->video_frame_rate() / (frc.factor() * TIME_HZ);
 364                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 365         }
 366
 367         /* XXX: don't seek audio because we don't need to... */
 368 }
 369
 370 void
 371 Player::setup_pieces ()
 372 {
 373         list<shared_ptr<Piece> > old_pieces = _pieces;
 374
 375         _pieces.clear ();
 376
 377         Playlist::ContentList content = _playlist->content ();
 378         sort (content.begin(), content.end(), ContentSorter ());
 379
 380         for (Playlist::ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 381
 382                 shared_ptr<Piece> piece (new Piece (*i));
 383
 384                 /* XXX: into content? */
 385
 386                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 387                 if (fc) {
 388                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 389
 390                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3));
 391                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 392                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1));
 393
 394                         piece->decoder = fd;
 395                 }
 396
 397                 shared_ptr<const ImageMagickContent> ic = dynamic_pointer_cast<const ImageMagickContent> (*i);
 398                 if (ic) {
 399                         shared_ptr<ImageMagickDecoder> id;
 400
 401                         /* See if we can re-use an old ImageMagickDecoder */
 402                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 403                                 shared_ptr<ImageMagickDecoder> imd = dynamic_pointer_cast<ImageMagickDecoder> ((*j)->decoder);
 404                                 if (imd && imd->content() == ic) {
 405                                         id = imd;
 406                                 }
 407                         }
 408
 409                         if (!id) {
 410                                 id.reset (new ImageMagickDecoder (_film, ic));
 411                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3));
 412                         }
 413
 414                         piece->decoder = id;
 415                 }
 416
 417                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 418                 if (sc) {
 419                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 420                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 421
 422                         piece->decoder = sd;
 423                 }
 424
 425                 _pieces.push_back (piece);
 426         }
 427
 428 #ifdef DEBUG_PLAYER
 429         cout << "=== Player setup:\n";
 430         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 431                 cout << *(i->get()) << "\n";
 432         }
 433 #endif
 434 }
 435
 436 void
 437 Player::content_changed (weak_ptr<Content> w, int p)
 438 {
 439         shared_ptr<Content> c = w.lock ();
 440         if (!c) {
 441                 return;
 442         }
 443
 444         if (
 445                 p == ContentProperty::START || p == ContentProperty::LENGTH ||
 446                 p == VideoContentProperty::VIDEO_CROP || p == VideoContentProperty::VIDEO_RATIO
 447                 ) {
 448
 449                 _have_valid_pieces = false;
 450                 Changed ();
 451         }
 452 }
 453
 454 void
 455 Player::playlist_changed ()
 456 {
 457         _have_valid_pieces = false;
 458         Changed ();
 459 }
 460
 461 void
 462 Player::set_video_container_size (libdcp::Size s)
 463 {
 464         _video_container_size = s;
 465         _black_frame.reset (new SimpleImage (PIX_FMT_RGB24, _video_container_size, true));
 466         _black_frame->make_black ();
 467 }
 468
 469 shared_ptr<Resampler>
 470 Player::resampler (shared_ptr<AudioContent> c)
 471 {
 472         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 473         if (i != _resamplers.end ()) {
 474                 return i->second;
 475         }
 476
 477         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 478         _resamplers[c] = r;
 479         return r;
 480 }
 481
 482 void
 483 Player::emit_black ()
 484 {
 485         /* XXX: use same here */
 486         Video (_black_frame, false, _video_position);
 487         _video_position += _film->video_frames_to_time (1);
 488 }
 489
 490 void
 491 Player::emit_silence (OutputAudioFrame most)
 492 {
 493         OutputAudioFrame N = min (most, _film->dcp_audio_frame_rate() / 2);
 494         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->dcp_audio_channels(), N));
 495         silence->make_silent ();
 496         Audio (silence, _audio_position);
 497         _audio_position += _film->audio_frames_to_time (N);
 498 }
 499
 500 void
 501 Player::film_changed (Film::Property p)
 502 {
 503         /* Here we should notice Film properties that affect our output, and
 504            alert listeners that our output now would be different to how it was
 505            last time we were run.
 506         */
 507
 508         if (
 509                 p == Film::SCALER || p == Film::WITH_SUBTITLES ||
 510                 p == Film::SUBTITLE_SCALE || p == Film::SUBTITLE_OFFSET ||
 511                 p == Film::CONTAINER
 512                 ) {
 513
 514                 Changed ();
 515         }
 516 }
 517
 518 void
 519 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<TimedSubtitle> sub)
 520 {
 521         shared_ptr<Piece> piece = weak_piece.lock ();
 522         if (!piece) {
 523                 return;
 524         }
 525
 526         _subtitle = sub;
 527         _subtitle_offset = piece->content->start ();
 528 }