src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "imagemagick_decoder.h"
  26 #include "imagemagick_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "playlist.h"
  30 #include "job.h"
  31 #include "image.h"
  32 #include "ratio.h"
  33 #include "resampler.h"
  34
  35 using std::list;
  36 using std::cout;
  37 using std::min;
  38 using std::max;
  39 using std::vector;
  40 using std::pair;
  41 using std::map;
  42 using boost::shared_ptr;
  43 using boost::weak_ptr;
  44 using boost::dynamic_pointer_cast;
  45
  46 #define DEBUG_PLAYER 1
  47
  48 class Piece
  49 {
  50 public:
  51         Piece (shared_ptr<Content> c)
  52                 : content (c)
  53                 , video_position (c->start ())
  54                 , audio_position (c->start ())
  55         {}
  56
  57         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  58                 : content (c)
  59                 , decoder (d)
  60                 , video_position (c->start ())
  61                 , audio_position (c->start ())
  62         {}
  63
  64         shared_ptr<Content> content;
  65         shared_ptr<Decoder> decoder;
  66         Time video_position;
  67         Time audio_position;
  68 };
  69
  70 #ifdef DEBUG_PLAYER
  71 std::ostream& operator<<(std::ostream& s, Piece const & p)
  72 {
  73         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
  74                 s << "\tffmpeg     ";
  75         } else if (dynamic_pointer_cast<ImageMagickContent> (p.content)) {
  76                 s << "\timagemagick";
  77         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
  78                 s << "\tsndfile    ";
  79         }
  80
  81         s << " at " << p.content->start() << " until " << p.content->end();
  82
  83         return s;
  84 }
  85 #endif
  86
  87 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  88         : _film (f)
  89         , _playlist (p)
  90         , _video (true)
  91         , _audio (true)
  92         , _have_valid_pieces (false)
  93         , _video_position (0)
  94         , _audio_position (0)
  95         , _audio_buffers (f->dcp_audio_channels(), 0)
  96 {
  97         _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  98         _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2));
  99         _film->Changed.connect (bind (&Player::film_changed, this, _1));
 100         set_video_container_size (_film->container()->size (_film->full_frame ()));
 101 }
 102
 103 void
 104 Player::disable_video ()
 105 {
 106         _video = false;
 107 }
 108
 109 void
 110 Player::disable_audio ()
 111 {
 112         _audio = false;
 113 }
 114
 115 bool
 116 Player::pass ()
 117 {
 118         if (!_have_valid_pieces) {
 119                 setup_pieces ();
 120                 _have_valid_pieces = true;
 121         }
 122
 123 #ifdef DEBUG_PLAYER
 124         cout << "= PASS\n";
 125 #endif
 126
 127         Time earliest_t = TIME_MAX;
 128         shared_ptr<Piece> earliest;
 129         enum {
 130                 VIDEO,
 131                 AUDIO
 132         } type = VIDEO;
 133
 134         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 135                 if ((*i)->decoder->done ()) {
 136                         continue;
 137                 }
 138
 139                 if (dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 140                         if ((*i)->video_position < earliest_t) {
 141                                 earliest_t = (*i)->video_position;
 142                                 earliest = *i;
 143                                 type = VIDEO;
 144                         }
 145                 }
 146
 147                 if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 148                         if ((*i)->audio_position < earliest_t) {
 149                                 earliest_t = (*i)->audio_position;
 150                                 earliest = *i;
 151                                 type = AUDIO;
 152                         }
 153                 }
 154         }
 155
 156         if (!earliest) {
 157 #ifdef DEBUG_PLAYER
 158                 cout << "no earliest piece.\n";
 159 #endif
 160
 161                 flush ();
 162                 return true;
 163         }
 164
 165         switch (type) {
 166         case VIDEO:
 167                 if (earliest_t > _video_position) {
 168 #ifdef DEBUG_PLAYER
 169                         cout << "no video here; emitting black frame.\n";
 170 #endif
 171                         emit_black ();
 172                 } else {
 173 #ifdef DEBUG_PLAYER
 174                         cout << "Pass " << *earliest << "\n";
 175 #endif
 176                         earliest->decoder->pass ();
 177                 }
 178                 break;
 179
 180         case AUDIO:
 181                 if (earliest_t > _audio_position) {
 182 #ifdef DEBUG_PLAYER
 183                         cout << "no audio here; emitting silence.\n";
 184 #endif
 185                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 186                 } else {
 187 #ifdef DEBUG_PLAYER
 188                         cout << "Pass " << *earliest << "\n";
 189 #endif
 190                         earliest->decoder->pass ();
 191                 }
 192                 break;
 193         }
 194
 195 #ifdef DEBUG_PLAYER
 196         cout << "\tpost pass " << _video_position << " " << _audio_position << "\n";
 197 #endif
 198
 199         return false;
 200 }
 201
 202 void
 203 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, bool same, VideoContent::Frame frame)
 204 {
 205         shared_ptr<Piece> piece = weak_piece.lock ();
 206         if (!piece) {
 207                 return;
 208         }
 209
 210         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 211         assert (content);
 212
 213         FrameRateConversion frc (content->video_frame_rate(), _film->dcp_video_frame_rate());
 214         if (frc.skip && (frame % 2) == 1) {
 215                 return;
 216         }
 217
 218         image = image->crop (content->crop(), true);
 219
 220         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
 221
 222         image = image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
 223
 224 #if 0
 225         if (film->with_subtitles ()) {
 226                 shared_ptr<Subtitle> sub;
 227                 if (_timed_subtitle && _timed_subtitle->displayed_at (t)) {
 228                         sub = _timed_subtitle->subtitle ();
 229                 }
 230
 231                 if (sub) {
 232                         dcpomatic::Rect const tx = subtitle_transformed_area (
 233                                 float (image_size.width) / content->video_size().width,
 234                                 float (image_size.height) / content->video_size().height,
 235                                 sub->area(), film->subtitle_offset(), film->subtitle_scale()
 236                                 );
 237
 238                         shared_ptr<Image> im = sub->image()->scale (tx.size(), film->scaler(), true);
 239                         image->alpha_blend (im, tx.position());
 240                 }
 241         }
 242 #endif
 243
 244         if (image_size != _video_container_size) {
 245                 assert (image_size.width <= _video_container_size.width);
 246                 assert (image_size.height <= _video_container_size.height);
 247                 shared_ptr<Image> im (new SimpleImage (PIX_FMT_RGB24, _video_container_size, true));
 248                 im->make_black ();
 249                 im->copy (image, Position ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 250                 image = im;
 251         }
 252
 253         Time time = content->start() + (frame * frc.factor() * TIME_HZ / _film->dcp_video_frame_rate());
 254
 255         Video (image, same, time);
 256         time += TIME_HZ / _film->dcp_video_frame_rate();
 257
 258         if (frc.repeat) {
 259                 Video (image, true, time);
 260                 time += TIME_HZ / _film->dcp_video_frame_rate();
 261         }
 262
 263         _video_position = piece->video_position = time;
 264 }
 265
 266 void
 267 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 268 {
 269         shared_ptr<Piece> piece = weak_piece.lock ();
 270         if (!piece) {
 271                 return;
 272         }
 273
 274         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 275         assert (content);
 276
 277         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 278                 audio = resampler(content)->run (audio);
 279         }
 280
 281         /* Remap channels */
 282         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->dcp_audio_channels(), audio->frames()));
 283         dcp_mapped->make_silent ();
 284         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 285         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 286                 dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 287         }
 288
 289         audio = dcp_mapped;
 290
 291         /* The time of this audio may indicate that some of our buffered audio is not going to
 292            be added to any more, so it can be emitted.
 293         */
 294
 295         Time const time = content->start() + (frame * TIME_HZ / _film->dcp_audio_frame_rate());
 296
 297         if (time > _audio_position) {
 298                 /* We can emit some audio from our buffers */
 299                 OutputAudioFrame const N = _film->time_to_audio_frames (time - _audio_position);
 300                 assert (N <= _audio_buffers.frames());
 301                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), N));
 302                 emit->copy_from (&_audio_buffers, N, 0, 0);
 303                 Audio (emit, _audio_position);
 304                 _audio_position = piece->audio_position = time + _film->audio_frames_to_time (N);
 305
 306                 /* And remove it from our buffers */
 307                 if (_audio_buffers.frames() > N) {
 308                         _audio_buffers.move (N, 0, _audio_buffers.frames() - N);
 309                 }
 310                 _audio_buffers.set_frames (_audio_buffers.frames() - N);
 311         }
 312
 313         /* Now accumulate the new audio into our buffers */
 314         _audio_buffers.ensure_size (_audio_buffers.frames() + audio->frames());
 315         _audio_buffers.accumulate_frames (audio.get(), 0, 0, audio->frames ());
 316         _audio_buffers.set_frames (_audio_buffers.frames() + audio->frames());
 317 }
 318
 319 void
 320 Player::flush ()
 321 {
 322         if (_audio_buffers.frames() > 0) {
 323                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), _audio_buffers.frames()));
 324                 emit->copy_from (&_audio_buffers, _audio_buffers.frames(), 0, 0);
 325                 Audio (emit, _audio_position);
 326                 _audio_position += _film->audio_frames_to_time (_audio_buffers.frames ());
 327                 _audio_buffers.set_frames (0);
 328         }
 329
 330         while (_video_position < _audio_position) {
 331                 emit_black ();
 332         }
 333
 334         while (_audio_position < _video_position) {
 335                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 336         }
 337
 338 }
 339
 340 /** @return true on error */
 341 void
 342 Player::seek (Time t, bool accurate)
 343 {
 344         if (!_have_valid_pieces) {
 345                 setup_pieces ();
 346                 _have_valid_pieces = true;
 347         }
 348
 349         if (_pieces.empty ()) {
 350                 return;
 351         }
 352
 353         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 354                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 355                 if (!vc) {
 356                         continue;
 357                 }
 358
 359                 Time s = t - vc->start ();
 360                 s = max (static_cast<Time> (0), s);
 361                 s = min (vc->length(), s);
 362
 363                 FrameRateConversion frc (vc->video_frame_rate(), _film->dcp_video_frame_rate());
 364                 VideoContent::Frame f = s * vc->video_frame_rate() / (frc.factor() * TIME_HZ);
 365                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 366         }
 367
 368         /* XXX: don't seek audio because we don't need to... */
 369 }
 370
 371 void
 372 Player::setup_pieces ()
 373 {
 374         list<shared_ptr<Piece> > old_pieces = _pieces;
 375
 376         _pieces.clear ();
 377
 378         Playlist::ContentList content = _playlist->content ();
 379         sort (content.begin(), content.end(), ContentSorter ());
 380
 381         for (Playlist::ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 382
 383                 shared_ptr<Piece> piece (new Piece (*i));
 384
 385                 /* XXX: into content? */
 386
 387                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 388                 if (fc) {
 389                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 390
 391                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3));
 392                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 393
 394                         piece->decoder = fd;
 395                 }
 396
 397                 shared_ptr<const ImageMagickContent> ic = dynamic_pointer_cast<const ImageMagickContent> (*i);
 398                 if (ic) {
 399                         shared_ptr<ImageMagickDecoder> id;
 400
 401                         /* See if we can re-use an old ImageMagickDecoder */
 402                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 403                                 shared_ptr<ImageMagickDecoder> imd = dynamic_pointer_cast<ImageMagickDecoder> ((*j)->decoder);
 404                                 if (imd && imd->content() == ic) {
 405                                         id = imd;
 406                                 }
 407                         }
 408
 409                         if (!id) {
 410                                 id.reset (new ImageMagickDecoder (_film, ic));
 411                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3));
 412                         }
 413
 414                         piece->decoder = id;
 415                 }
 416
 417                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 418                 if (sc) {
 419                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 420                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 421
 422                         piece->decoder = sd;
 423                 }
 424
 425                 _pieces.push_back (piece);
 426         }
 427
 428 #ifdef DEBUG_PLAYER
 429         cout << "=== Player setup:\n";
 430         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 431                 cout << *(i->get()) << "\n";
 432         }
 433 #endif
 434 }
 435
 436 void
 437 Player::content_changed (weak_ptr<Content> w, int p)
 438 {
 439         shared_ptr<Content> c = w.lock ();
 440         if (!c) {
 441                 return;
 442         }
 443
 444         if (
 445                 p == ContentProperty::START || p == ContentProperty::LENGTH ||
 446                 p == VideoContentProperty::VIDEO_CROP || p == VideoContentProperty::VIDEO_RATIO
 447                 ) {
 448
 449                 _have_valid_pieces = false;
 450                 Changed ();
 451         }
 452 }
 453
 454 void
 455 Player::playlist_changed ()
 456 {
 457         _have_valid_pieces = false;
 458         Changed ();
 459 }
 460
 461 void
 462 Player::set_video_container_size (libdcp::Size s)
 463 {
 464         _video_container_size = s;
 465         _black_frame.reset (new SimpleImage (PIX_FMT_RGB24, _video_container_size, true));
 466         _black_frame->make_black ();
 467 }
 468
 469 shared_ptr<Resampler>
 470 Player::resampler (shared_ptr<AudioContent> c)
 471 {
 472         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 473         if (i != _resamplers.end ()) {
 474                 return i->second;
 475         }
 476
 477         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 478         _resamplers[c] = r;
 479         return r;
 480 }
 481
 482 void
 483 Player::emit_black ()
 484 {
 485         /* XXX: use same here */
 486         Video (_black_frame, false, _video_position);
 487         _video_position += _film->video_frames_to_time (1);
 488 }
 489
 490 void
 491 Player::emit_silence (OutputAudioFrame most)
 492 {
 493         OutputAudioFrame N = min (most, _film->dcp_audio_frame_rate() / 2);
 494         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->dcp_audio_channels(), N));
 495         silence->make_silent ();
 496         Audio (silence, _audio_position);
 497         _audio_position += _film->audio_frames_to_time (N);
 498 }
 499
 500 void
 501 Player::film_changed (Film::Property p)
 502 {
 503         /* Here we should notice Film properties that affect our output, and
 504            alert listeners that our output now would be different to how it was
 505            last time we were run.
 506         */
 507
 508         if (
 509                 p == Film::SCALER || p == Film::WITH_SUBTITLES ||
 510                 p == Film::SUBTITLE_SCALE || p == Film::SUBTITLE_OFFSET ||
 511                 p == Film::CONTAINER
 512                 ) {
 513
 514                 Changed ();
 515         }
 516 }