src/lib/player.cc

   1 /*
   2     Copyright (C) 2013 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include "player.h"
  22 #include "film.h"
  23 #include "ffmpeg_decoder.h"
  24 #include "ffmpeg_content.h"
  25 #include "imagemagick_decoder.h"
  26 #include "imagemagick_content.h"
  27 #include "sndfile_decoder.h"
  28 #include "sndfile_content.h"
  29 #include "playlist.h"
  30 #include "job.h"
  31 #include "image.h"
  32 #include "ratio.h"
  33 #include "resampler.h"
  34
  35 using std::list;
  36 using std::cout;
  37 using std::min;
  38 using std::max;
  39 using std::vector;
  40 using std::pair;
  41 using std::map;
  42 using boost::shared_ptr;
  43 using boost::weak_ptr;
  44 using boost::dynamic_pointer_cast;
  45
  46 #define DEBUG_PLAYER 1
  47
  48 class Piece
  49 {
  50 public:
  51         Piece (shared_ptr<Content> c)
  52                 : content (c)
  53                 , video_position (c->start ())
  54                 , audio_position (c->start ())
  55         {}
  56
  57         Piece (shared_ptr<Content> c, shared_ptr<Decoder> d)
  58                 : content (c)
  59                 , decoder (d)
  60                 , video_position (c->start ())
  61                 , audio_position (c->start ())
  62         {}
  63
  64         shared_ptr<Content> content;
  65         shared_ptr<Decoder> decoder;
  66         Time video_position;
  67         Time audio_position;
  68 };
  69
  70 #ifdef DEBUG_PLAYER
  71 std::ostream& operator<<(std::ostream& s, Piece const & p)
  72 {
  73         if (dynamic_pointer_cast<FFmpegContent> (p.content)) {
  74                 s << "\tffmpeg     ";
  75         } else if (dynamic_pointer_cast<ImageMagickContent> (p.content)) {
  76                 s << "\timagemagick";
  77         } else if (dynamic_pointer_cast<SndfileContent> (p.content)) {
  78                 s << "\tsndfile    ";
  79         }
  80
  81         s << " at " << p.content->start() << " until " << p.content->end();
  82
  83         return s;
  84 }
  85 #endif
  86
  87 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  88         : _film (f)
  89         , _playlist (p)
  90         , _video (true)
  91         , _audio (true)
  92         , _have_valid_pieces (false)
  93         , _video_position (0)
  94         , _audio_position (0)
  95         , _audio_buffers (f->dcp_audio_channels(), 0)
  96 {
  97         _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  98         _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2));
  99         _film->Changed.connect (bind (&Player::film_changed, this, _1));
 100         set_video_container_size (_film->container()->size (_film->full_frame ()));
 101 }
 102
 103 void
 104 Player::disable_video ()
 105 {
 106         _video = false;
 107 }
 108
 109 void
 110 Player::disable_audio ()
 111 {
 112         _audio = false;
 113 }
 114
 115 bool
 116 Player::pass ()
 117 {
 118         if (!_have_valid_pieces) {
 119                 setup_pieces ();
 120                 _have_valid_pieces = true;
 121         }
 122
 123 #ifdef DEBUG_PLAYER
 124         cout << "= PASS " << this << "\n";
 125 #endif
 126
 127         Time earliest_t = TIME_MAX;
 128         shared_ptr<Piece> earliest;
 129         enum {
 130                 VIDEO,
 131                 AUDIO
 132         } type = VIDEO;
 133
 134         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 135                 if ((*i)->decoder->done ()) {
 136                         continue;
 137                 }
 138
 139                 if (dynamic_pointer_cast<VideoDecoder> ((*i)->decoder)) {
 140                         if ((*i)->video_position < earliest_t) {
 141                                 earliest_t = (*i)->video_position;
 142                                 earliest = *i;
 143                                 type = VIDEO;
 144                         }
 145                 }
 146
 147                 if (dynamic_pointer_cast<AudioDecoder> ((*i)->decoder)) {
 148                         if ((*i)->audio_position < earliest_t) {
 149                                 earliest_t = (*i)->audio_position;
 150                                 earliest = *i;
 151                                 type = AUDIO;
 152                         }
 153                 }
 154         }
 155
 156         if (!earliest) {
 157 #ifdef DEBUG_PLAYER
 158                 cout << "no earliest piece.\n";
 159 #endif
 160
 161                 flush ();
 162                 return true;
 163         }
 164
 165         switch (type) {
 166         case VIDEO:
 167                 if (earliest_t > _video_position) {
 168 #ifdef DEBUG_PLAYER
 169                         cout << "no video here; emitting black frame.\n";
 170 #endif
 171                         emit_black ();
 172                 } else {
 173 #ifdef DEBUG_PLAYER
 174                         cout << "Pass " << *earliest << "\n";
 175 #endif
 176                         earliest->decoder->pass ();
 177                 }
 178                 break;
 179
 180         case AUDIO:
 181                 if (earliest_t > _audio_position) {
 182 #ifdef DEBUG_PLAYER
 183                         cout << "no audio here; emitting silence.\n";
 184 #endif
 185                         emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position));
 186                 } else {
 187 #ifdef DEBUG_PLAYER
 188                         cout << "Pass " << *earliest << "\n";
 189 #endif
 190                         earliest->decoder->pass ();
 191                 }
 192                 break;
 193         }
 194
 195 #ifdef DEBUG_PLAYER
 196         cout << "\tpost pass " << _video_position << " " << _audio_position << "\n";
 197 #endif
 198
 199         return false;
 200 }
 201
 202 void
 203 Player::process_video (weak_ptr<Piece> weak_piece, shared_ptr<const Image> image, bool same, VideoContent::Frame frame)
 204 {
 205         shared_ptr<Piece> piece = weak_piece.lock ();
 206         if (!piece) {
 207                 return;
 208         }
 209
 210         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 211         assert (content);
 212
 213         FrameRateConversion frc (content->video_frame_rate(), _film->dcp_video_frame_rate());
 214         if (frc.skip && (frame % 2) == 1) {
 215                 return;
 216         }
 217
 218         image = image->crop (content->crop(), true);
 219
 220         libdcp::Size const image_size = content->ratio()->size (_video_container_size);
 221
 222         image = image->scale_and_convert_to_rgb (image_size, _film->scaler(), true);
 223
 224 #if 0
 225         if (film->with_subtitles ()) {
 226                 shared_ptr<Subtitle> sub;
 227                 if (_timed_subtitle && _timed_subtitle->displayed_at (t)) {
 228                         sub = _timed_subtitle->subtitle ();
 229                 }
 230
 231                 if (sub) {
 232                         dcpomatic::Rect const tx = subtitle_transformed_area (
 233                                 float (image_size.width) / content->video_size().width,
 234                                 float (image_size.height) / content->video_size().height,
 235                                 sub->area(), film->subtitle_offset(), film->subtitle_scale()
 236                                 );
 237
 238                         shared_ptr<Image> im = sub->image()->scale (tx.size(), film->scaler(), true);
 239                         image->alpha_blend (im, tx.position());
 240                 }
 241         }
 242 #endif
 243
 244         if (image_size != _video_container_size) {
 245                 assert (image_size.width <= _video_container_size.width);
 246                 assert (image_size.height <= _video_container_size.height);
 247                 shared_ptr<Image> im (new SimpleImage (PIX_FMT_RGB24, _video_container_size, true));
 248                 im->make_black ();
 249                 im->copy (image, Position ((_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.height) / 2));
 250                 image = im;
 251         }
 252
 253         Time time = content->start() + (frame * frc.factor() * TIME_HZ / _film->dcp_video_frame_rate());
 254
 255         Video (image, same, time);
 256         time += TIME_HZ / _film->dcp_video_frame_rate();
 257
 258         if (frc.repeat) {
 259                 Video (image, true, time);
 260                 time += TIME_HZ / _film->dcp_video_frame_rate();
 261         }
 262
 263         _video_position = piece->video_position = time;
 264 }
 265
 266 void
 267 Player::process_audio (weak_ptr<Piece> weak_piece, shared_ptr<const AudioBuffers> audio, AudioContent::Frame frame)
 268 {
 269         shared_ptr<Piece> piece = weak_piece.lock ();
 270         if (!piece) {
 271                 return;
 272         }
 273
 274         shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> (piece->content);
 275         assert (content);
 276
 277         if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) {
 278                 audio = resampler(content)->run (audio);
 279         }
 280
 281         /* Remap channels */
 282         shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->dcp_audio_channels(), audio->frames()));
 283         dcp_mapped->make_silent ();
 284         list<pair<int, libdcp::Channel> > map = content->audio_mapping().content_to_dcp ();
 285         for (list<pair<int, libdcp::Channel> >::iterator i = map.begin(); i != map.end(); ++i) {
 286                 dcp_mapped->accumulate_channel (audio.get(), i->first, i->second);
 287         }
 288
 289         /* The time of this audio may indicate that some of our buffered audio is not going to
 290            be added to any more, so it can be emitted.
 291         */
 292
 293         Time const time = content->start() + (frame * TIME_HZ / _film->dcp_audio_frame_rate());
 294
 295         if (time > _audio_position) {
 296                 /* We can emit some audio from our buffers */
 297                 OutputAudioFrame const N = _film->time_to_audio_frames (time - _audio_position);
 298                 assert (N <= _audio_buffers.frames());
 299                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), N));
 300                 emit->copy_from (&_audio_buffers, N, 0, 0);
 301                 Audio (emit, _audio_position);
 302                 _audio_position = piece->audio_position = time + _film->audio_frames_to_time (N);
 303
 304                 /* And remove it from our buffers */
 305                 if (_audio_buffers.frames() > N) {
 306                         _audio_buffers.move (N, 0, _audio_buffers.frames() - N);
 307                 }
 308                 _audio_buffers.set_frames (_audio_buffers.frames() - N);
 309         }
 310
 311         /* Now accumulate the new audio into our buffers */
 312         _audio_buffers.ensure_size (_audio_buffers.frames() + audio->frames());
 313         _audio_buffers.accumulate_frames (audio.get(), 0, 0, audio->frames ());
 314         _audio_buffers.set_frames (_audio_buffers.frames() + audio->frames());
 315 }
 316
 317 void
 318 Player::flush ()
 319 {
 320         if (_audio_buffers.frames() > 0) {
 321                 shared_ptr<AudioBuffers> emit (new AudioBuffers (_audio_buffers.channels(), _audio_buffers.frames()));
 322                 emit->copy_from (&_audio_buffers, _audio_buffers.frames(), 0, 0);
 323                 Audio (emit, _audio_position);
 324                 _audio_position += _film->audio_frames_to_time (_audio_buffers.frames ());
 325                 _audio_buffers.set_frames (0);
 326         }
 327
 328         while (_video_position < _audio_position) {
 329                 emit_black ();
 330         }
 331
 332         while (_audio_position < _video_position) {
 333                 emit_silence (_film->time_to_audio_frames (_video_position - _audio_position));
 334         }
 335
 336 }
 337
 338 /** @return true on error */
 339 void
 340 Player::seek (Time t, bool accurate)
 341 {
 342         if (!_have_valid_pieces) {
 343                 setup_pieces ();
 344                 _have_valid_pieces = true;
 345         }
 346
 347         if (_pieces.empty ()) {
 348                 return;
 349         }
 350
 351         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 352                 shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> ((*i)->content);
 353                 if (!vc) {
 354                         continue;
 355                 }
 356
 357                 Time s = t - vc->start ();
 358                 s = max (static_cast<Time> (0), s);
 359                 s = min (vc->length(), s);
 360
 361                 FrameRateConversion frc (vc->video_frame_rate(), _film->dcp_video_frame_rate());
 362                 VideoContent::Frame f = s * vc->video_frame_rate() / (frc.factor() * TIME_HZ);
 363                 dynamic_pointer_cast<VideoDecoder>((*i)->decoder)->seek (f, accurate);
 364         }
 365
 366         /* XXX: don't seek audio because we don't need to... */
 367 }
 368
 369 void
 370 Player::setup_pieces ()
 371 {
 372         list<shared_ptr<Piece> > old_pieces = _pieces;
 373
 374         _pieces.clear ();
 375
 376         Playlist::ContentList content = _playlist->content ();
 377         sort (content.begin(), content.end(), ContentSorter ());
 378
 379         for (Playlist::ContentList::iterator i = content.begin(); i != content.end(); ++i) {
 380
 381                 shared_ptr<Piece> piece (new Piece (*i));
 382
 383                 /* XXX: into content? */
 384
 385                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 386                 if (fc) {
 387                         shared_ptr<FFmpegDecoder> fd (new FFmpegDecoder (_film, fc, _video, _audio));
 388
 389                         fd->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3));
 390                         fd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 391
 392                         piece->decoder = fd;
 393                 }
 394
 395                 shared_ptr<const ImageMagickContent> ic = dynamic_pointer_cast<const ImageMagickContent> (*i);
 396                 if (ic) {
 397                         shared_ptr<ImageMagickDecoder> id;
 398
 399                         /* See if we can re-use an old ImageMagickDecoder */
 400                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 401                                 shared_ptr<ImageMagickDecoder> imd = dynamic_pointer_cast<ImageMagickDecoder> ((*j)->decoder);
 402                                 if (imd && imd->content() == ic) {
 403                                         id = imd;
 404                                 }
 405                         }
 406
 407                         if (!id) {
 408                                 id.reset (new ImageMagickDecoder (_film, ic));
 409                                 id->Video.connect (bind (&Player::process_video, this, piece, _1, _2, _3));
 410                         }
 411
 412                         piece->decoder = id;
 413                 }
 414
 415                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 416                 if (sc) {
 417                         shared_ptr<AudioDecoder> sd (new SndfileDecoder (_film, sc));
 418                         sd->Audio.connect (bind (&Player::process_audio, this, piece, _1, _2));
 419
 420                         piece->decoder = sd;
 421                 }
 422
 423                 _pieces.push_back (piece);
 424         }
 425
 426 #ifdef DEBUG_PLAYER
 427         cout << "=== Player setup:\n";
 428         for (list<shared_ptr<Piece> >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) {
 429                 cout << *(i->get()) << "\n";
 430         }
 431 #endif
 432 }
 433
 434 void
 435 Player::content_changed (weak_ptr<Content> w, int p)
 436 {
 437         shared_ptr<Content> c = w.lock ();
 438         if (!c) {
 439                 return;
 440         }
 441
 442         if (
 443                 p == ContentProperty::START || p == ContentProperty::LENGTH ||
 444                 p == VideoContentProperty::VIDEO_CROP || p == VideoContentProperty::VIDEO_RATIO
 445                 ) {
 446
 447                 _have_valid_pieces = false;
 448                 Changed ();
 449         }
 450 }
 451
 452 void
 453 Player::playlist_changed ()
 454 {
 455         _have_valid_pieces = false;
 456         Changed ();
 457 }
 458
 459 void
 460 Player::set_video_container_size (libdcp::Size s)
 461 {
 462         _video_container_size = s;
 463         _black_frame.reset (new SimpleImage (PIX_FMT_RGB24, _video_container_size, true));
 464         _black_frame->make_black ();
 465 }
 466
 467 shared_ptr<Resampler>
 468 Player::resampler (shared_ptr<AudioContent> c)
 469 {
 470         map<shared_ptr<AudioContent>, shared_ptr<Resampler> >::iterator i = _resamplers.find (c);
 471         if (i != _resamplers.end ()) {
 472                 return i->second;
 473         }
 474
 475         shared_ptr<Resampler> r (new Resampler (c->content_audio_frame_rate(), c->output_audio_frame_rate(), c->audio_channels()));
 476         _resamplers[c] = r;
 477         return r;
 478 }
 479
 480 void
 481 Player::emit_black ()
 482 {
 483         /* XXX: use same here */
 484         Video (_black_frame, false, _video_position);
 485         _video_position += _film->video_frames_to_time (1);
 486 }
 487
 488 void
 489 Player::emit_silence (OutputAudioFrame most)
 490 {
 491         OutputAudioFrame N = min (most, _film->dcp_audio_frame_rate() / 2);
 492         shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->dcp_audio_channels(), N));
 493         silence->make_silent ();
 494         Audio (silence, _audio_position);
 495         _audio_position += _film->audio_frames_to_time (N);
 496 }
 497
 498 void
 499 Player::film_changed (Film::Property p)
 500 {
 501         /* Here we should notice Film properties that affect our output, and
 502            alert listeners that our output now would be different to how it was
 503            last time we were run.
 504         */
 505
 506         if (
 507                 p == Film::SCALER || p == Film::WITH_SUBTITLES ||
 508                 p == Film::SUBTITLE_SCALE || p == Film::SUBTITLE_OFFSET ||
 509                 p == Film::CONTAINER
 510                 ) {
 511
 512                 Changed ();
 513         }
 514 }