src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "imagemagick_decoder.h"
  26 #include "imagemagick_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "playlist.h"
  30 #include "job.h"
  31 #include "image.h"
  32 #include "ratio.h"
  33 #include "resampler.h"
  34
  35 using std::list;
  36 using std::cout;
  37 using std::min;
  38 using std::max;
  39 using std::vector;
  40 using std::pair;
  41 using std::map;
  42 using boost::shared_ptr;
  43 using boost::weak_ptr;
  44 using boost::dynamic_pointer_cast;
  45
  46 #define DEBUG_PLAYER 1
  47
  48 class Piece
  49 {
  50 public:
  51         Piece (shared_ptr<Content> c)
  52                 : content (c)
  53                 , video_position (c->start ())
  54                 , audio_position (c->start ())
  55         {}
  56
  57         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  58                 : content (c)
  59                 , decoder (d)
  60                 , video_position (c->start ())
  61                 , audio_position (c->start ())
  62         {}
  63
  64         shared_ptr<Content> content;
  65         shared_ptr<Decoder> decoder;
  66         Time video_position;
  67         Time audio_position;
  68 };
  69
  70 #ifdef DEBUG_PLAYER
  71 std::ostream& operator<<(std::ostream& s, Piece const & p)
  72 {
  73         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
  74                 s << "\tffmpeg     ";
  75         } else if (dynamic_pointer_cast<ImageMagickContent> (p.content)) {
  76                 s << "\timagemagick";
  77         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
  78                 s << "\tsndfile    ";
  79         }
  80
  81         s << " at " << p.content->start() << " until " << p.content->end();
  82
  83         return s;
  84 }
  85 #endif
  86
  87 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  88         : _film (f)
  89         , _playlist (p)
  90         , _video (true)
  91         , _audio (true)
  92         , _have_valid_pieces (false)
  93         , _video_position (0)
  94         , _audio_position (0)
  95         , _audio_buffers (f->dcp_audio_channels(), 0)
  96 {
  97         _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  98         _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2));
  99         _film->Changed.connect (bind (&Player::film_changed, this, _1));
 100         set_video_container_size (_film->container()->size (_film->full_frame ()));
 101 }
 102
 103 void
 104 Player::disable_video ()
 105 {
 106         _video = false;
 107 }
 108
 109 void
 110 Player::disable_audio ()
 111 {
 112         _audio = false;
 113 }
 114
 115 bool
 116 Player::pass ()
 117 {
 118         if (!_have_valid_pieces) {
 119                 setup_pieces ();
 120                 _have_valid_pieces = true;
 121         }
 122
 123 #ifdef DEBUG_PLAYER
 124         cout << "= PASS " << this << "\n";
 125 #endif
 126
 127         Time earliest_t = TIME_MAX;
 128         shared_ptr<Piece> earliest;
 129         enum {
 130                 VIDEO,
 131                 AUDIO
 132         } type = VIDEO;
 133
 134         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 135                 if ((*i)->decoder->done ()) {
 136                         continue;
 137                 }
 138
 139                 if (dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 140                         if ((*i)->video_position < earliest_t) {
 141                                 earliest_t = (*i)->video_position;
 142                                 earliest = *i;
 143                                 type = VIDEO;
 144                         }
 145                 }
 146
 147                 if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 148                         if ((*i)->audio_position < earliest_t) {
 149                                 earliest_t = (*i)->audio_position;
 150                                 earliest = *i;
 151                                 type = AUDIO;
 152                         }
 153                 }
 154         }
 155
 156         if (!earliest) {
 157 #ifdef DEBUG_PLAYER
 158                 cout << "no earliest piece.\n";
 159 #endif
 160
 161                 flush ();
 162                 return true;
 163         }
 164
 165         switch (type) {
 166         case VIDEO:
 167                 if (earliest_t > _video_position) {
 168 #ifdef DEBUG_PLAYER
 169                         cout << "no video here; emitting black frame.\n";
 170 #endif
 171                         emit_black ();
 172                 } else {
 173 #ifdef DEBUG_PLAYER
 174                         cout << "Pass " << *earliest << "\n";
 175 #endif
 176                         earliest->decoder->pass ();
 177                 }
 178                 break;
 179
 180         case AUDIO:
 181                 if (earliest_t > _audio_position) {
 182 #ifdef DEBUG_PLAYER
 183                         cout << "no audio here; emitting silence.\n";
 184 #endif
 185                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 186                 } else {
 187 #ifdef DEBUG_PLAYER
 188                         cout << "Pass " << *earliest << "\n";
 189 #endif
 190                         earliest->decoder->pass ();
 191                 }
 192                 break;
 193         }
 194
 195 #ifdef DEBUG_PLAYER
 196         cout << "\tpost pass " << _video_position << " " << _audio_position << "\n";
 197 #endif
 198
 199         return false;
 200 }
 201
 202 void
 203 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, bool same, VideoContent::Frame frame)
 204 {
 205         shared_ptr<Piece> piece = weak_piece.lock ();
 206         if (!piece) {
 207                 return;
 208         }
 209
 210         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 211         assert (content);
 212
 213         FrameRateConversion frc (content->video_frame_rate(), _film->dcp_video_frame_rate());
 214         if (frc.skip && (frame % 2) == 1) {
 215                 return;
 216         }
 217
 218         image = image->crop (content->crop(), true);
 219
 220         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
 221
 222         image = image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
 223
 224 #if 0
 225         if (film->with_subtitles ()) {
 226                 shared_ptr<Subtitle> sub;
 227                 if (_timed_subtitle && _timed_subtitle->displayed_at (t)) {
 228                         sub = _timed_subtitle->subtitle ();
 229                 }
 230
 231                 if (sub) {
 232                         dcpomatic::Rect const tx = subtitle_transformed_area (
 233                                 float (image_size.width) / content->video_size().width,
 234                                 float (image_size.height) / content->video_size().height,
 235                                 sub->area(), film->subtitle_offset(), film->subtitle_scale()
 236                                 );
 237
 238                         shared_ptr<Image> im = sub->image()->scale (tx.size(), film->scaler(), true);
 239                         image->alpha_blend (im, tx.position());
 240                 }
 241         }
 242 #endif
 243
 244         if (image_size != _video_container_size) {
 245                 assert (image_size.width <= _video_container_size.width);
 246                 assert (image_size.height <= _video_container_size.height);
 247                 shared_ptr<Image> im (new SimpleImage (PIX_FMT_RGB24, _video_container_size, true));
 248                 im->make_black ();
 249                 im->copy (image, Position ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 250                 image = im;
 251         }
 252
 253         Time time = content->start() + (frame * frc.factor() * TIME_HZ / _film->dcp_video_frame_rate());
 254
 255         Video (image, same, time);
 256         time += TIME_HZ / _film->dcp_video_frame_rate();
 257
 258         if (frc.repeat) {
 259                 Video (image, true, time);
 260                 time += TIME_HZ / _film->dcp_video_frame_rate();
 261         }
 262
 263         _video_position = piece->video_position = time;
 264 }
 265
 266 void
 267 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 268 {
 269         shared_ptr<Piece> piece = weak_piece.lock ();
 270         if (!piece) {
 271                 return;
 272         }
 273
 274         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 275         assert (content);
 276
 277         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 278                 audio = resampler(content)->run (audio);
 279         }
 280
 281         /* Remap channels */
 282         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->dcp_audio_channels(), audio->frames()));
 283         dcp_mapped->make_silent ();
 284         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 285         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 286                 dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 287         }
 288
 289         /* The time of this audio may indicate that some of our buffered audio is not going to
 290            be added to any more, so it can be emitted.
 291         */
 292
 293         Time const time = content->start() + (frame * TIME_HZ / _film->dcp_audio_frame_rate());
 294
 295         if (time > _audio_position) {
 296                 /* We can emit some audio from our buffers */
 297                 OutputAudioFrame const N = _film->time_to_audio_frames (time - _audio_position);
 298                 assert (N <= _audio_buffers.frames());
 299                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), N));
 300                 emit->copy_from (&_audio_buffers, N, 0, 0);
 301                 Audio (emit, _audio_position);
 302                 _audio_position = piece->audio_position = time + _film->audio_frames_to_time (N);
 303
 304                 /* And remove it from our buffers */
 305                 if (_audio_buffers.frames() > N) {
 306                         _audio_buffers.move (N, 0, _audio_buffers.frames() - N);
 307                 }
 308                 _audio_buffers.set_frames (_audio_buffers.frames() - N);
 309         }
 310
 311         /* Now accumulate the new audio into our buffers */
 312         _audio_buffers.ensure_size (_audio_buffers.frames() + audio->frames());
 313         _audio_buffers.accumulate_frames (audio.get(), 0, 0, audio->frames ());
 314         _audio_buffers.set_frames (_audio_buffers.frames() + audio->frames());
 315 }
 316
 317 void
 318 Player::flush ()
 319 {
 320         if (_audio_buffers.frames() > 0) {
 321                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), _audio_buffers.frames()));
 322                 emit->copy_from (&_audio_buffers, _audio_buffers.frames(), 0, 0);
 323                 Audio (emit, _audio_position);
 324                 _audio_position += _film->audio_frames_to_time (_audio_buffers.frames ());
 325                 _audio_buffers.set_frames (0);
 326         }
 327
 328         while (_video_position < _audio_position) {
 329                 emit_black ();
 330         }
 331
 332         while (_audio_position < _video_position) {
 333                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 334         }
 335
 336 }
 337
 338 /** @return true on error */
 339 void
 340 Player::seek (Time t)
 341 {
 342         if (!_have_valid_pieces) {
 343                 setup_pieces ();
 344                 _have_valid_pieces = true;
 345         }
 346
 347         if (_pieces.empty ()) {
 348                 return;
 349         }
 350
 351         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 352                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 353                 if (!vc) {
 354                         continue;
 355                 }
 356
 357                 Time s = t - vc->start ();
 358                 s = max (static_cast<Time> (0), s);
 359                 s = min (vc->length(), s);
 360
 361                 FrameRateConversion frc (vc->video_frame_rate(), _film->dcp_video_frame_rate());
 362                 VideoContent::Frame f = s * _film->dcp_video_frame_rate() / (frc.factor() * TIME_HZ);
 363                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f);
 364         }
 365
 366         /* XXX: don't seek audio because we don't need to... */
 367 }
 368
 369
 370 void
 371 Player::seek_back ()
 372 {
 373
 374 }
 375
 376 void
 377 Player::setup_pieces ()
 378 {
 379         list<shared_ptr<Piece> > old_pieces = _pieces;
 380
 381         _pieces.clear ();
 382
 383         Playlist::ContentList content = _playlist->content ();
 384         sort (content.begin(), content.end(), ContentSorter ());
 385
 386         for (Playlist::ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 387
 388                 shared_ptr<Piece> piece (new Piece (*i));
 389
 390                 /* XXX: into content? */
 391
 392                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 393                 if (fc) {
 394                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 395
 396                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3));
 397                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 398
 399                         piece->decoder = fd;
 400                 }
 401
 402                 shared_ptr<const ImageMagickContent> ic = dynamic_pointer_cast<const ImageMagickContent> (*i);
 403                 if (ic) {
 404                         shared_ptr<ImageMagickDecoder> id;
 405
 406                         /* See if we can re-use an old ImageMagickDecoder */
 407                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 408                                 shared_ptr<ImageMagickDecoder> imd = dynamic_pointer_cast<ImageMagickDecoder> ((*j)->decoder);
 409                                 if (imd && imd->content() == ic) {
 410                                         id = imd;
 411                                 }
 412                         }
 413
 414                         if (!id) {
 415                                 id.reset (new ImageMagickDecoder (_film, ic));
 416                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3));
 417                         }
 418
 419                         piece->decoder = id;
 420                 }
 421
 422                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 423                 if (sc) {
 424                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 425                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 426
 427                         piece->decoder = sd;
 428                 }
 429
 430                 _pieces.push_back (piece);
 431         }
 432
 433 #ifdef DEBUG_PLAYER
 434         cout << "=== Player setup:\n";
 435         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 436                 cout << *(i->get()) << "\n";
 437         }
 438 #endif
 439 }
 440
 441 void
 442 Player::content_changed (weak_ptr<Content> w, int p)
 443 {
 444         shared_ptr<Content> c = w.lock ();
 445         if (!c) {
 446                 return;
 447         }
 448
 449         if (
 450                 p == ContentProperty::START || p == ContentProperty::LENGTH ||
 451                 p == VideoContentProperty::VIDEO_CROP || p == VideoContentProperty::VIDEO_RATIO
 452                 ) {
 453
 454                 _have_valid_pieces = false;
 455                 Changed ();
 456         }
 457 }
 458
 459 void
 460 Player::playlist_changed ()
 461 {
 462         _have_valid_pieces = false;
 463         Changed ();
 464 }
 465
 466 void
 467 Player::set_video_container_size (libdcp::Size s)
 468 {
 469         _video_container_size = s;
 470         _black_frame.reset (new SimpleImage (PIX_FMT_RGB24, _video_container_size, true));
 471         _black_frame->make_black ();
 472 }
 473
 474 shared_ptr<Resampler>
 475 Player::resampler (shared_ptr<AudioContent> c)
 476 {
 477         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 478         if (i != _resamplers.end ()) {
 479                 return i->second;
 480         }
 481
 482         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 483         _resamplers[c] = r;
 484         return r;
 485 }
 486
 487 void
 488 Player::emit_black ()
 489 {
 490         /* XXX: use same here */
 491         Video (_black_frame, false, _video_position);
 492         _video_position += _film->video_frames_to_time (1);
 493 }
 494
 495 void
 496 Player::emit_silence (OutputAudioFrame most)
 497 {
 498         OutputAudioFrame N = min (most, _film->dcp_audio_frame_rate() / 2);
 499         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->dcp_audio_channels(), N));
 500         silence->make_silent ();
 501         Audio (silence, _audio_position);
 502         _audio_position += _film->audio_frames_to_time (N);
 503 }
 504
 505 void
 506 Player::film_changed (Film::Property p)
 507 {
 508         /* Here we should notice Film properties that affect our output, and
 509            alert listeners that our output now would be different to how it was
 510            last time we were run.
 511         */
 512
 513         if (
 514                 p == Film::SCALER || p == Film::WITH_SUBTITLES ||
 515                 p == Film::SUBTITLE_SCALE || p == Film::SUBTITLE_OFFSET ||
 516                 p == Film::CONTAINER
 517                 ) {
 518
 519                 Changed ();
 520         }
 521 }