src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include <algorithm>
  22 #include "player.h"
  23 #include "film.h"
  24 #include "ffmpeg_decoder.h"
  25 #include "audio_buffers.h"
  26 #include "ffmpeg_content.h"
  27 #include "image_decoder.h"
  28 #include "image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "subrip_decoder.h"
  33 #include "subrip_content.h"
  34 #include "playlist.h"
  35 #include "job.h"
  36 #include "image.h"
  37 #include "ratio.h"
  38 #include "log.h"
  39 #include "scaler.h"
  40 #include "render_subtitles.h"
  41 #include "dcp_video.h"
  42 #include "config.h"
  43 #include "content_video.h"
  44
  45 using std::list;
  46 using std::cout;
  47 using std::min;
  48 using std::max;
  49 using std::min;
  50 using std::vector;
  51 using std::pair;
  52 using std::map;
  53 using std::make_pair;
  54 using boost::shared_ptr;
  55 using boost::weak_ptr;
  56 using boost::dynamic_pointer_cast;
  57 using boost::optional;
  58
  59 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  60         : _film (f)
  61         , _playlist (p)
  62         , _have_valid_pieces (false)
  63         , _approximate_size (false)
  64         , _burn_subtitles (false)
  65 {
  66         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  67         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  68         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  69         set_video_container_size (_film->frame_size ());
  70 }
  71
  72 void
  73 Player::setup_pieces ()
  74 {
  75         list<shared_ptr<Piece> > old_pieces = _pieces;
  76         _pieces.clear ();
  77
  78         ContentList content = _playlist->content ();
  79
  80         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
  81
  82                 if (!(*i)->paths_valid ()) {
  83                         continue;
  84                 }
  85
  86                 shared_ptr<Decoder> decoder;
  87                 optional<FrameRateChange> frc;
  88
  89                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
  90                 DCPTime best_overlap_t;
  91                 shared_ptr<VideoContent> best_overlap;
  92                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
  93                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
  94                         if (!vc) {
  95                                 continue;
  96                         }
  97
  98                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
  99                         if (overlap > best_overlap_t) {
 100                                 best_overlap = vc;
 101                                 best_overlap_t = overlap;
 102                         }
 103                 }
 104
 105                 optional<FrameRateChange> best_overlap_frc;
 106                 if (best_overlap) {
 107                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 108                 } else {
 109                         /* No video overlap; e.g. if the DCP is just audio */
 110                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 111                 }
 112
 113                 /* FFmpeg */
 114                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 115                 if (fc) {
 116                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
 117                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 118                 }
 119
 120                 /* ImageContent */
 121                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 122                 if (ic) {
 123                         /* See if we can re-use an old ImageDecoder */
 124                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 125                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 126                                 if (imd && imd->content() == ic) {
 127                                         decoder = imd;
 128                                 }
 129                         }
 130
 131                         if (!decoder) {
 132                                 decoder.reset (new ImageDecoder (ic));
 133                         }
 134
 135                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 136                 }
 137
 138                 /* SndfileContent */
 139                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 140                 if (sc) {
 141                         decoder.reset (new SndfileDecoder (sc));
 142                         frc = best_overlap_frc;
 143                 }
 144
 145                 /* SubRipContent */
 146                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
 147                 if (rc) {
 148                         decoder.reset (new SubRipDecoder (rc));
 149                         frc = best_overlap_frc;
 150                 }
 151
 152                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 153         }
 154
 155         _have_valid_pieces = true;
 156 }
 157
 158 void
 159 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 160 {
 161         shared_ptr<Content> c = w.lock ();
 162         if (!c) {
 163                 return;
 164         }
 165
 166         if (
 167                 property == ContentProperty::POSITION ||
 168                 property == ContentProperty::LENGTH ||
 169                 property == ContentProperty::TRIM_START ||
 170                 property == ContentProperty::TRIM_END ||
 171                 property == ContentProperty::PATH ||
 172                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 173                 ) {
 174
 175                 _have_valid_pieces = false;
 176                 Changed (frequent);
 177
 178         } else if (
 179                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 180                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 181                 property == SubtitleContentProperty::SUBTITLE_SCALE ||
 182                 property == VideoContentProperty::VIDEO_CROP ||
 183                 property == VideoContentProperty::VIDEO_SCALE ||
 184                 property == VideoContentProperty::VIDEO_FRAME_RATE
 185                 ) {
 186
 187                 Changed (frequent);
 188         }
 189 }
 190
 191 void
 192 Player::playlist_changed ()
 193 {
 194         _have_valid_pieces = false;
 195         Changed (false);
 196 }
 197
 198 void
 199 Player::set_video_container_size (dcp::Size s)
 200 {
 201         _video_container_size = s;
 202
 203         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 204         _black_image->make_black ();
 205 }
 206
 207 void
 208 Player::film_changed (Film::Property p)
 209 {
 210         /* Here we should notice Film properties that affect our output, and
 211            alert listeners that our output now would be different to how it was
 212            last time we were run.
 213         */
 214
 215         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 216                 Changed (false);
 217         }
 218 }
 219
 220 list<PositionImage>
 221 Player::process_content_image_subtitles (shared_ptr<SubtitleContent> content, list<shared_ptr<ContentImageSubtitle> > subs) const
 222 {
 223         list<PositionImage> all;
 224
 225         for (list<shared_ptr<ContentImageSubtitle> >::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 226                 if (!(*i)->image) {
 227                         continue;
 228                 }
 229
 230                 dcpomatic::Rect<double> in_rect = (*i)->rectangle;
 231                 dcp::Size scaled_size;
 232
 233                 in_rect.x += content->subtitle_x_offset ();
 234                 in_rect.y += content->subtitle_y_offset ();
 235
 236                 /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 237                 scaled_size.width = in_rect.width * _video_container_size.width * content->subtitle_scale ();
 238                 scaled_size.height = in_rect.height * _video_container_size.height * content->subtitle_scale ();
 239
 240                 /* Then we need a corrective translation, consisting of two parts:
 241                  *
 242                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 243                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 244                  *
 245                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 246                  *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 247                  *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 248                  *
 249                  * Combining these two translations gives these expressions.
 250                  */
 251
 252                 all.push_back (
 253                         PositionImage (
 254                                 (*i)->image->scale (
 255                                         scaled_size,
 256                                         Scaler::from_id ("bicubic"),
 257                                         (*i)->image->pixel_format (),
 258                                         true
 259                                         ),
 260                                 Position<int> (
 261                                         rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - content->subtitle_scale ()) / 2))),
 262                                         rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - content->subtitle_scale ()) / 2)))
 263                                         )
 264                                 )
 265                         );
 266         }
 267
 268         return all;
 269 }
 270
 271 list<PositionImage>
 272 Player::process_content_text_subtitles (list<shared_ptr<ContentTextSubtitle> > sub) const
 273 {
 274         list<PositionImage> all;
 275         for (list<shared_ptr<ContentTextSubtitle> >::const_iterator i = sub.begin(); i != sub.end(); ++i) {
 276                 if (!(*i)->subs.empty ()) {
 277                         all.push_back (render_subtitles ((*i)->subs, _video_container_size));
 278                 }
 279         }
 280
 281         return all;
 282 }
 283
 284 void
 285 Player::set_approximate_size ()
 286 {
 287         _approximate_size = true;
 288 }
 289
 290 shared_ptr<DCPVideo>
 291 Player::black_dcp_video (DCPTime time) const
 292 {
 293         return shared_ptr<DCPVideo> (
 294                 new DCPVideo (
 295                         _black_image,
 296                         EYES_BOTH,
 297                         Crop (),
 298                         _video_container_size,
 299                         _video_container_size,
 300                         Scaler::from_id ("bicubic"),
 301                         Config::instance()->colour_conversions().front().conversion,
 302                         time
 303                 )
 304         );
 305 }
 306
 307 shared_ptr<DCPVideo>
 308 Player::content_to_dcp (
 309         shared_ptr<VideoContent> content,
 310         ContentVideo content_video,
 311         list<shared_ptr<Piece> > subs,
 312         DCPTime time,
 313         dcp::Size image_size) const
 314 {
 315         shared_ptr<DCPVideo> dcp_video (
 316                 new DCPVideo (
 317                         content_video.image,
 318                         content_video.eyes,
 319                         content->crop (),
 320                         image_size,
 321                         _video_container_size,
 322                         _film->scaler(),
 323                         content->colour_conversion (),
 324                         time
 325                         )
 326                 );
 327
 328
 329         /* Add subtitles */
 330
 331         list<PositionImage> sub_images;
 332
 333         for (list<shared_ptr<Piece> >::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 334                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*i)->decoder);
 335                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*i)->content);
 336                 ContentTime const from = dcp_to_content_subtitle (*i, time);
 337                 ContentTime const to = from + ContentTime::from_frames (1, content->video_frame_rate ());
 338
 339                 list<shared_ptr<ContentImageSubtitle> > image_subtitles = subtitle_decoder->get_image_subtitles (from, to);
 340                 if (!image_subtitles.empty ()) {
 341                         list<PositionImage> im = process_content_image_subtitles (
 342                                 subtitle_content,
 343                                 image_subtitles
 344                                 );
 345
 346                         copy (im.begin(), im.end(), back_inserter (sub_images));
 347                 }
 348
 349                 if (_burn_subtitles) {
 350                         list<shared_ptr<ContentTextSubtitle> > text_subtitles = subtitle_decoder->get_text_subtitles (from, to);
 351                         if (!text_subtitles.empty ()) {
 352                                 list<PositionImage> im = process_content_text_subtitles (text_subtitles);
 353                                 copy (im.begin(), im.end(), back_inserter (sub_images));
 354                         }
 355                 }
 356         }
 357
 358         if (!sub_images.empty ()) {
 359                 dcp_video->set_subtitle (merge (sub_images));
 360         }
 361
 362         return dcp_video;
 363 }
 364
 365 /** @return All DCPVideo at the given time (there may be two frames for 3D) */
 366 list<shared_ptr<DCPVideo> >
 367 Player::get_video (DCPTime time, bool accurate)
 368 {
 369         if (!_have_valid_pieces) {
 370                 setup_pieces ();
 371         }
 372
 373         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
 374                 time,
 375                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
 376                 );
 377
 378         list<shared_ptr<DCPVideo> > dcp_video;
 379
 380         if (ov.empty ()) {
 381                 /* No video content at this time */
 382                 dcp_video.push_back (black_dcp_video (time));
 383                 return dcp_video;
 384         }
 385
 386         /* Create a DCPVideo from the content's video at this time */
 387
 388         shared_ptr<Piece> piece = ov.back ();
 389         shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
 390         assert (decoder);
 391         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 392         assert (content);
 393
 394         list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
 395         if (content_video.empty ()) {
 396                 dcp_video.push_back (black_dcp_video (time));
 397                 return dcp_video;
 398         }
 399
 400         dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 401         if (_approximate_size) {
 402                 image_size.width &= ~3;
 403                 image_size.height &= ~3;
 404         }
 405
 406         for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
 407                 list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (
 408                         time,
 409                         time + DCPTime::from_frames (1, _film->video_frame_rate ())
 410                         );
 411
 412                 dcp_video.push_back (content_to_dcp (content, *i, subs, time, image_size));
 413         }
 414
 415         return dcp_video;
 416 }
 417
 418 shared_ptr<AudioBuffers>
 419 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
 420 {
 421         if (!_have_valid_pieces) {
 422                 setup_pieces ();
 423         }
 424
 425         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
 426
 427         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
 428         audio->make_silent ();
 429
 430         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
 431         if (ov.empty ()) {
 432                 return audio;
 433         }
 434
 435         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
 436
 437                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
 438                 assert (content);
 439                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 440                 assert (decoder);
 441
 442                 if (content->audio_frame_rate() == 0) {
 443                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
 444                          * audio stream).
 445                          */
 446                         continue;
 447                 }
 448
 449                 /* The time that we should request from the content */
 450                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
 451                 DCPTime offset;
 452                 if (request < DCPTime ()) {
 453                         /* We went off the start of the content, so we will need to offset
 454                            the stuff we get back.
 455                         */
 456                         offset = -request;
 457                         request = DCPTime ();
 458                 }
 459
 460                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
 461
 462                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
 463                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, length_frames, accurate);
 464
 465                 /* Gain */
 466                 if (content->audio_gain() != 0) {
 467                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
 468                         gain->apply_gain (content->audio_gain ());
 469                         all->audio = gain;
 470                 }
 471
 472                 /* Remap channels */
 473                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
 474                 dcp_mapped->make_silent ();
 475                 AudioMapping map = content->audio_mapping ();
 476                 for (int i = 0; i < map.content_channels(); ++i) {
 477                         for (int j = 0; j < _film->audio_channels(); ++j) {
 478                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 479                                         dcp_mapped->accumulate_channel (
 480                                                 all->audio.get(),
 481                                                 i,
 482                                                 j,
 483                                                 map.get (i, static_cast<dcp::Channel> (j))
 484                                                 );
 485                                 }
 486                         }
 487                 }
 488
 489                 all->audio = dcp_mapped;
 490
 491                 audio->accumulate_frames (
 492                         all->audio.get(),
 493                         content_frame - all->frame,
 494                         offset.frames (_film->audio_frame_rate()),
 495                         min (AudioFrame (all->audio->frames()), length_frames) - offset.frames (_film->audio_frame_rate ())
 496                         );
 497         }
 498
 499         return audio;
 500 }
 501
 502 VideoFrame
 503 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 504 {
 505         /* s is the offset of t from the start position of this content */
 506         DCPTime s = t - piece->content->position ();
 507         s = DCPTime (max (int64_t (0), s.get ()));
 508         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 509
 510         /* Convert this to the content frame */
 511         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) * piece->frc.factor ();
 512 }
 513
 514 AudioFrame
 515 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
 516 {
 517         /* s is the offset of t from the start position of this content */
 518         DCPTime s = t - piece->content->position ();
 519         s = DCPTime (max (int64_t (0), s.get ()));
 520         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 521
 522         /* Convert this to the content frame */
 523         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
 524 }
 525
 526 ContentTime
 527 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
 528 {
 529         /* s is the offset of t from the start position of this content */
 530         DCPTime s = t - piece->content->position ();
 531         s = DCPTime (max (int64_t (0), s.get ()));
 532         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 533
 534         return ContentTime (s, piece->frc);
 535 }
 536
 537 void
 538 PlayerStatistics::dump (shared_ptr<Log> log) const
 539 {
 540         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat));
 541         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()));
 542 }
 543
 544 PlayerStatistics const &
 545 Player::statistics () const
 546 {
 547         return _statistics;
 548 }