src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "imagemagick_decoder.h"
  26 #include "imagemagick_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "subtitle_content.h"
  30 #include "playlist.h"
  31 #include "job.h"
  32 #include "image.h"
  33 #include "ratio.h"
  34 #include "resampler.h"
  35 #include "subtitle.h"
  36
  37 using std::list;
  38 using std::cout;
  39 using std::min;
  40 using std::max;
  41 using std::vector;
  42 using std::pair;
  43 using std::map;
  44 using boost::shared_ptr;
  45 using boost::weak_ptr;
  46 using boost::dynamic_pointer_cast;
  47
  48 #define DEBUG_PLAYER 1
  49
  50 class Piece
  51 {
  52 public:
  53         Piece (shared_ptr<Content> c)
  54                 : content (c)
  55                 , video_position (c->start ())
  56                 , audio_position (c->start ())
  57         {}
  58
  59         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  60                 : content (c)
  61                 , decoder (d)
  62                 , video_position (c->start ())
  63                 , audio_position (c->start ())
  64         {}
  65
  66         shared_ptr<Content> content;
  67         shared_ptr<Decoder> decoder;
  68         Time video_position;
  69         Time audio_position;
  70 };
  71
  72 #ifdef DEBUG_PLAYER
  73 std::ostream& operator<<(std::ostream& s, Piece const & p)
  74 {
  75         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
  76                 s << "\tffmpeg     ";
  77         } else if (dynamic_pointer_cast<ImageMagickContent> (p.content)) {
  78                 s << "\timagemagick";
  79         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
  80                 s << "\tsndfile    ";
  81         }
  82
  83         s << " at " << p.content->start() << " until " << p.content->end();
  84
  85         return s;
  86 }
  87 #endif
  88
  89 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  90         : _film (f)
  91         , _playlist (p)
  92         , _video (true)
  93         , _audio (true)
  94         , _have_valid_pieces (false)
  95         , _video_position (0)
  96         , _audio_position (0)
  97         , _audio_buffers (f->dcp_audio_channels(), 0)
  98 {
  99         _playlist->Changed.connect (bind (&Player::playlist_changed, this));
 100         _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2));
 101         _film->Changed.connect (bind (&Player::film_changed, this, _1));
 102         set_video_container_size (_film->container()->size (_film->full_frame ()));
 103 }
 104
 105 void
 106 Player::disable_video ()
 107 {
 108         _video = false;
 109 }
 110
 111 void
 112 Player::disable_audio ()
 113 {
 114         _audio = false;
 115 }
 116
 117 bool
 118 Player::pass ()
 119 {
 120         if (!_have_valid_pieces) {
 121                 setup_pieces ();
 122                 _have_valid_pieces = true;
 123         }
 124
 125 #ifdef DEBUG_PLAYER
 126         cout << "= PASS\n";
 127 #endif
 128
 129         Time earliest_t = TIME_MAX;
 130         shared_ptr<Piece> earliest;
 131         enum {
 132                 VIDEO,
 133                 AUDIO
 134         } type = VIDEO;
 135
 136         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 137                 if ((*i)->decoder->done ()) {
 138                         continue;
 139                 }
 140
 141                 if (dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 142                         if ((*i)->video_position < earliest_t) {
 143                                 earliest_t = (*i)->video_position;
 144                                 earliest = *i;
 145                                 type = VIDEO;
 146                         }
 147                 }
 148
 149                 if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 150                         if ((*i)->audio_position < earliest_t) {
 151                                 earliest_t = (*i)->audio_position;
 152                                 earliest = *i;
 153                                 type = AUDIO;
 154                         }
 155                 }
 156         }
 157
 158         if (!earliest) {
 159 #ifdef DEBUG_PLAYER
 160                 cout << "no earliest piece.\n";
 161 #endif
 162
 163                 flush ();
 164                 return true;
 165         }
 166
 167         switch (type) {
 168         case VIDEO:
 169                 if (earliest_t > _video_position) {
 170 #ifdef DEBUG_PLAYER
 171                         cout << "no video here; emitting black frame.\n";
 172 #endif
 173                         emit_black ();
 174                 } else {
 175 #ifdef DEBUG_PLAYER
 176                         cout << "Pass " << *earliest << "\n";
 177 #endif
 178                         earliest->decoder->pass ();
 179                 }
 180                 break;
 181
 182         case AUDIO:
 183                 if (earliest_t > _audio_position) {
 184 #ifdef DEBUG_PLAYER
 185                         cout << "no audio here; emitting silence.\n";
 186 #endif
 187                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 188                 } else {
 189 #ifdef DEBUG_PLAYER
 190                         cout << "Pass " << *earliest << "\n";
 191 #endif
 192                         earliest->decoder->pass ();
 193                 }
 194                 break;
 195         }
 196
 197 #ifdef DEBUG_PLAYER
 198         cout << "\tpost pass " << _video_position << " " << _audio_position << "\n";
 199 #endif
 200
 201         return false;
 202 }
 203
 204 void
 205 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, bool same, VideoContent::Frame frame)
 206 {
 207         shared_ptr<Piece> piece = weak_piece.lock ();
 208         if (!piece) {
 209                 return;
 210         }
 211
 212         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 213         assert (content);
 214
 215         FrameRateConversion frc (content->video_frame_rate(), _film->dcp_video_frame_rate());
 216         if (frc.skip && (frame % 2) == 1) {
 217                 return;
 218         }
 219
 220         shared_ptr<Image> work_image = image->crop (content->crop(), true);
 221
 222         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
 223
 224         work_image = work_image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
 225
 226         Time time = content->start() + (frame * frc.factor() * TIME_HZ / _film->dcp_video_frame_rate());
 227
 228         if (_film->with_subtitles ()) {
 229                 shared_ptr<Subtitle> sub;
 230                 if (_subtitle && _subtitle->displayed_at (time - _subtitle_content_time)) {
 231                         sub = _subtitle->subtitle ();
 232                 }
 233
 234                 if (sub) {
 235                         dcpomatic::Rect const tx = subtitle_transformed_area (
 236                                 float (image_size.width) / content->video_size().width,
 237                                 float (image_size.height) / content->video_size().height,
 238                                 sub->area(), _subtitle_offset, _subtitle_scale
 239                                 );
 240
 241                         shared_ptr<Image> im = sub->image()->scale (tx.size(), _film->scaler(), true);
 242                         work_image->alpha_blend (im, tx.position());
 243                 }
 244         }
 245
 246         if (image_size != _video_container_size) {
 247                 assert (image_size.width <= _video_container_size.width);
 248                 assert (image_size.height <= _video_container_size.height);
 249                 shared_ptr<Image> im (new SimpleImage (PIX_FMT_RGB24, _video_container_size, true));
 250                 im->make_black ();
 251                 im->copy (work_image, Position ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 252                 work_image = im;
 253         }
 254
 255         Video (work_image, same, time);
 256         time += TIME_HZ / _film->dcp_video_frame_rate();
 257
 258         if (frc.repeat) {
 259                 Video (work_image, true, time);
 260                 time += TIME_HZ / _film->dcp_video_frame_rate();
 261         }
 262
 263         _video_position = piece->video_position = time;
 264 }
 265
 266 void
 267 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 268 {
 269         shared_ptr<Piece> piece = weak_piece.lock ();
 270         if (!piece) {
 271                 return;
 272         }
 273
 274         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 275         assert (content);
 276
 277         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 278                 audio = resampler(content)->run (audio);
 279         }
 280
 281         /* Remap channels */
 282         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->dcp_audio_channels(), audio->frames()));
 283         dcp_mapped->make_silent ();
 284         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 285         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 286                 dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 287         }
 288
 289         audio = dcp_mapped;
 290
 291         /* The time of this audio may indicate that some of our buffered audio is not going to
 292            be added to any more, so it can be emitted.
 293         */
 294
 295         Time const time = content->start() + (frame * TIME_HZ / _film->dcp_audio_frame_rate());
 296
 297         if (time > _audio_position) {
 298                 /* We can emit some audio from our buffers */
 299                 OutputAudioFrame const N = _film->time_to_audio_frames (time - _audio_position);
 300                 assert (N <= _audio_buffers.frames());
 301                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), N));
 302                 emit->copy_from (&_audio_buffers, N, 0, 0);
 303                 Audio (emit, _audio_position);
 304                 _audio_position = piece->audio_position = time + _film->audio_frames_to_time (N);
 305
 306                 /* And remove it from our buffers */
 307                 if (_audio_buffers.frames() > N) {
 308                         _audio_buffers.move (N, 0, _audio_buffers.frames() - N);
 309                 }
 310                 _audio_buffers.set_frames (_audio_buffers.frames() - N);
 311         }
 312
 313         /* Now accumulate the new audio into our buffers */
 314         _audio_buffers.ensure_size (_audio_buffers.frames() + audio->frames());
 315         _audio_buffers.accumulate_frames (audio.get(), 0, 0, audio->frames ());
 316         _audio_buffers.set_frames (_audio_buffers.frames() + audio->frames());
 317 }
 318
 319 void
 320 Player::flush ()
 321 {
 322         if (_audio_buffers.frames() > 0) {
 323                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), _audio_buffers.frames()));
 324                 emit->copy_from (&_audio_buffers, _audio_buffers.frames(), 0, 0);
 325                 Audio (emit, _audio_position);
 326                 _audio_position += _film->audio_frames_to_time (_audio_buffers.frames ());
 327                 _audio_buffers.set_frames (0);
 328         }
 329
 330         while (_video_position < _audio_position) {
 331                 emit_black ();
 332         }
 333
 334         while (_audio_position < _video_position) {
 335                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 336         }
 337
 338 }
 339
 340 /** @return true on error */
 341 void
 342 Player::seek (Time t, bool accurate)
 343 {
 344         if (!_have_valid_pieces) {
 345                 setup_pieces ();
 346                 _have_valid_pieces = true;
 347         }
 348
 349         if (_pieces.empty ()) {
 350                 return;
 351         }
 352
 353         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 354                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 355                 if (!vc) {
 356                         continue;
 357                 }
 358
 359                 Time s = t - vc->start ();
 360                 s = max (static_cast<Time> (0), s);
 361                 s = min (vc->length(), s);
 362
 363                 FrameRateConversion frc (vc->video_frame_rate(), _film->dcp_video_frame_rate());
 364                 VideoContent::Frame f = s * vc->video_frame_rate() / (frc.factor() * TIME_HZ);
 365                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 366         }
 367
 368         /* XXX: don't seek audio because we don't need to... */
 369 }
 370
 371 void
 372 Player::setup_pieces ()
 373 {
 374         list<shared_ptr<Piece> > old_pieces = _pieces;
 375
 376         _pieces.clear ();
 377
 378         Playlist::ContentList content = _playlist->content ();
 379         sort (content.begin(), content.end(), ContentSorter ());
 380
 381         for (Playlist::ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 382
 383                 shared_ptr<Piece> piece (new Piece (*i));
 384
 385                 /* XXX: into content? */
 386
 387                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 388                 if (fc) {
 389                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 390
 391                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3));
 392                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 393                         fd->Subtitle.connect (bind (&Player::process_subtitle, this, piece, _1));
 394
 395                         piece->decoder = fd;
 396                 }
 397
 398                 shared_ptr<const ImageMagickContent> ic = dynamic_pointer_cast<const ImageMagickContent> (*i);
 399                 if (ic) {
 400                         shared_ptr<ImageMagickDecoder> id;
 401
 402                         /* See if we can re-use an old ImageMagickDecoder */
 403                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 404                                 shared_ptr<ImageMagickDecoder> imd = dynamic_pointer_cast<ImageMagickDecoder> ((*j)->decoder);
 405                                 if (imd && imd->content() == ic) {
 406                                         id = imd;
 407                                 }
 408                         }
 409
 410                         if (!id) {
 411                                 id.reset (new ImageMagickDecoder (_film, ic));
 412                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3));
 413                         }
 414
 415                         piece->decoder = id;
 416                 }
 417
 418                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 419                 if (sc) {
 420                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 421                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 422
 423                         piece->decoder = sd;
 424                 }
 425
 426                 _pieces.push_back (piece);
 427         }
 428
 429 #ifdef DEBUG_PLAYER
 430         cout << "=== Player setup:\n";
 431         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 432                 cout << *(i->get()) << "\n";
 433         }
 434 #endif
 435 }
 436
 437 void
 438 Player::content_changed (weak_ptr<Content> w, int p)
 439 {
 440         shared_ptr<Content> c = w.lock ();
 441         if (!c) {
 442                 return;
 443         }
 444
 445         if (
 446                 p == ContentProperty::START || p == ContentProperty::LENGTH ||
 447                 p == VideoContentProperty::VIDEO_CROP || p == VideoContentProperty::VIDEO_RATIO
 448                 ) {
 449
 450                 _have_valid_pieces = false;
 451                 Changed ();
 452         }
 453 }
 454
 455 void
 456 Player::playlist_changed ()
 457 {
 458         _have_valid_pieces = false;
 459         Changed ();
 460 }
 461
 462 void
 463 Player::set_video_container_size (libdcp::Size s)
 464 {
 465         _video_container_size = s;
 466         _black_frame.reset (new SimpleImage (PIX_FMT_RGB24, _video_container_size, true));
 467         _black_frame->make_black ();
 468 }
 469
 470 shared_ptr<Resampler>
 471 Player::resampler (shared_ptr<AudioContent> c)
 472 {
 473         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 474         if (i != _resamplers.end ()) {
 475                 return i->second;
 476         }
 477
 478         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 479         _resamplers[c] = r;
 480         return r;
 481 }
 482
 483 void
 484 Player::emit_black ()
 485 {
 486         /* XXX: use same here */
 487         Video (_black_frame, false, _video_position);
 488         _video_position += _film->video_frames_to_time (1);
 489 }
 490
 491 void
 492 Player::emit_silence (OutputAudioFrame most)
 493 {
 494         OutputAudioFrame N = min (most, _film->dcp_audio_frame_rate() / 2);
 495         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->dcp_audio_channels(), N));
 496         silence->make_silent ();
 497         Audio (silence, _audio_position);
 498         _audio_position += _film->audio_frames_to_time (N);
 499 }
 500
 501 void
 502 Player::film_changed (Film::Property p)
 503 {
 504         /* Here we should notice Film properties that affect our output, and
 505            alert listeners that our output now would be different to how it was
 506            last time we were run.
 507         */
 508
 509         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER) {
 510                 Changed ();
 511         }
 512 }
 513
 514 void
 515 Player::process_subtitle (weak_ptr<Piece> weak_piece, shared_ptr<TimedSubtitle> sub)
 516 {
 517         shared_ptr<Piece> piece = weak_piece.lock ();
 518         if (!piece) {
 519                 return;
 520         }
 521
 522         shared_ptr<SubtitleContent> sc = dynamic_pointer_cast<SubtitleContent> (piece->content);
 523         assert (sc);
 524
 525         _subtitle = sub;
 526         _subtitle_content_time = piece->content->start ();
 527         _subtitle_offset = sc->subtitle_offset ();
 528         _subtitle_scale = sc->subtitle_scale ();
 529 }