src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include <algorithm>
  22 #include "player.h"
  23 #include "film.h"
  24 #include "ffmpeg_decoder.h"
  25 #include "audio_buffers.h"
  26 #include "ffmpeg_content.h"
  27 #include "image_decoder.h"
  28 #include "image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "subrip_decoder.h"
  33 #include "subrip_content.h"
  34 #include "playlist.h"
  35 #include "job.h"
  36 #include "image.h"
  37 #include "ratio.h"
  38 #include "log.h"
  39 #include "scaler.h"
  40 #include "render_subtitles.h"
  41 #include "dcp_video.h"
  42 #include "config.h"
  43 #include "content_video.h"
  44
  45 using std::list;
  46 using std::cout;
  47 using std::min;
  48 using std::max;
  49 using std::min;
  50 using std::vector;
  51 using std::pair;
  52 using std::map;
  53 using std::make_pair;
  54 using boost::shared_ptr;
  55 using boost::weak_ptr;
  56 using boost::dynamic_pointer_cast;
  57 using boost::optional;
  58
  59 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  60         : _film (f)
  61         , _playlist (p)
  62         , _have_valid_pieces (false)
  63         , _approximate_size (false)
  64         , _burn_subtitles (false)
  65 {
  66         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  67         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  68         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  69         set_video_container_size (_film->frame_size ());
  70 }
  71
  72 void
  73 Player::setup_pieces ()
  74 {
  75         list<shared_ptr<Piece> > old_pieces = _pieces;
  76         _pieces.clear ();
  77
  78         ContentList content = _playlist->content ();
  79
  80         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
  81
  82                 if (!(*i)->paths_valid ()) {
  83                         continue;
  84                 }
  85
  86                 shared_ptr<Decoder> decoder;
  87                 optional<FrameRateChange> frc;
  88
  89                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
  90                 DCPTime best_overlap_t;
  91                 shared_ptr<VideoContent> best_overlap;
  92                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
  93                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
  94                         if (!vc) {
  95                                 continue;
  96                         }
  97
  98                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
  99                         if (overlap > best_overlap_t) {
 100                                 best_overlap = vc;
 101                                 best_overlap_t = overlap;
 102                         }
 103                 }
 104
 105                 optional<FrameRateChange> best_overlap_frc;
 106                 if (best_overlap) {
 107                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 108                 } else {
 109                         /* No video overlap; e.g. if the DCP is just audio */
 110                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 111                 }
 112
 113                 /* FFmpeg */
 114                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 115                 if (fc) {
 116                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
 117                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 118                 }
 119
 120                 /* ImageContent */
 121                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 122                 if (ic) {
 123                         /* See if we can re-use an old ImageDecoder */
 124                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 125                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 126                                 if (imd && imd->content() == ic) {
 127                                         decoder = imd;
 128                                 }
 129                         }
 130
 131                         if (!decoder) {
 132                                 decoder.reset (new ImageDecoder (ic));
 133                         }
 134
 135                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 136                 }
 137
 138                 /* SndfileContent */
 139                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 140                 if (sc) {
 141                         decoder.reset (new SndfileDecoder (sc));
 142                         frc = best_overlap_frc;
 143                 }
 144
 145                 /* SubRipContent */
 146                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
 147                 if (rc) {
 148                         decoder.reset (new SubRipDecoder (rc));
 149                         frc = best_overlap_frc;
 150                 }
 151
 152                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 153         }
 154
 155         _have_valid_pieces = true;
 156 }
 157
 158 void
 159 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 160 {
 161         shared_ptr<Content> c = w.lock ();
 162         if (!c) {
 163                 return;
 164         }
 165
 166         if (
 167                 property == ContentProperty::POSITION ||
 168                 property == ContentProperty::LENGTH ||
 169                 property == ContentProperty::TRIM_START ||
 170                 property == ContentProperty::TRIM_END ||
 171                 property == ContentProperty::PATH ||
 172                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 173                 ) {
 174
 175                 _have_valid_pieces = false;
 176                 Changed (frequent);
 177
 178         } else if (
 179                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 180                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 181                 property == SubtitleContentProperty::SUBTITLE_SCALE ||
 182                 property == VideoContentProperty::VIDEO_CROP ||
 183                 property == VideoContentProperty::VIDEO_SCALE ||
 184                 property == VideoContentProperty::VIDEO_FRAME_RATE
 185                 ) {
 186
 187                 Changed (frequent);
 188         }
 189 }
 190
 191 void
 192 Player::playlist_changed ()
 193 {
 194         _have_valid_pieces = false;
 195         Changed (false);
 196 }
 197
 198 void
 199 Player::set_video_container_size (dcp::Size s)
 200 {
 201         _video_container_size = s;
 202
 203         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 204         _black_image->make_black ();
 205 }
 206
 207 void
 208 Player::film_changed (Film::Property p)
 209 {
 210         /* Here we should notice Film properties that affect our output, and
 211            alert listeners that our output now would be different to how it was
 212            last time we were run.
 213         */
 214
 215         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 216                 Changed (false);
 217         }
 218 }
 219
 220 list<PositionImage>
 221 Player::process_content_image_subtitles (shared_ptr<SubtitleContent> content, list<shared_ptr<ContentImageSubtitle> > subs)
 222 {
 223         list<PositionImage> all;
 224
 225         for (list<shared_ptr<ContentImageSubtitle> >::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 226                 if (!(*i)->image) {
 227                         continue;
 228                 }
 229
 230                 dcpomatic::Rect<double> in_rect = (*i)->rectangle;
 231                 dcp::Size scaled_size;
 232
 233                 in_rect.x += content->subtitle_x_offset ();
 234                 in_rect.y += content->subtitle_y_offset ();
 235
 236                 /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 237                 scaled_size.width = in_rect.width * _video_container_size.width * content->subtitle_scale ();
 238                 scaled_size.height = in_rect.height * _video_container_size.height * content->subtitle_scale ();
 239
 240                 /* Then we need a corrective translation, consisting of two parts:
 241                  *
 242                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 243                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 244                  *
 245                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 246                  *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 247                  *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 248                  *
 249                  * Combining these two translations gives these expressions.
 250                  */
 251
 252                 all.push_back (
 253                         PositionImage (
 254                                 (*i)->image->scale (
 255                                         scaled_size,
 256                                         Scaler::from_id ("bicubic"),
 257                                         (*i)->image->pixel_format (),
 258                                         true
 259                                         ),
 260                                 Position<int> (
 261                                         rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - content->subtitle_scale ()) / 2))),
 262                                         rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - content->subtitle_scale ()) / 2)))
 263                                         )
 264                                 )
 265                         );
 266         }
 267
 268         return all;
 269 }
 270
 271 list<PositionImage>
 272 Player::process_content_text_subtitles (list<shared_ptr<ContentTextSubtitle> > sub)
 273 {
 274         list<PositionImage> all;
 275         for (list<shared_ptr<ContentTextSubtitle> >::const_iterator i = sub.begin(); i != sub.end(); ++i) {
 276                 if (!(*i)->subs.empty ()) {
 277                         all.push_back (render_subtitles ((*i)->subs, _video_container_size));
 278                 }
 279         }
 280
 281         return all;
 282 }
 283
 284 void
 285 Player::set_approximate_size ()
 286 {
 287         _approximate_size = true;
 288 }
 289
 290 shared_ptr<DCPVideo>
 291 Player::black_dcp_video (DCPTime time) const
 292 {
 293         return shared_ptr<DCPVideo> (
 294                 new DCPVideo (
 295                         _black_image,
 296                         EYES_BOTH,
 297                         Crop (),
 298                         _video_container_size,
 299                         _video_container_size,
 300                         Scaler::from_id ("bicubic"),
 301                         Config::instance()->colour_conversions().front().conversion,
 302                         time
 303                 )
 304         );
 305 }
 306
 307 shared_ptr<DCPVideo>
 308 Player::get_video (DCPTime time, bool accurate)
 309 {
 310         if (!_have_valid_pieces) {
 311                 setup_pieces ();
 312         }
 313
 314         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
 315                 time,
 316                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
 317                 );
 318
 319         if (ov.empty ()) {
 320                 /* No video content at this time */
 321                 return black_dcp_video (time);
 322         }
 323
 324         /* Create a DCPVideo from the content's video at this time */
 325
 326         shared_ptr<Piece> piece = ov.back ();
 327         shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
 328         assert (decoder);
 329         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 330         assert (content);
 331
 332         optional<ContentVideo> dec = decoder->get_video (dcp_to_content_video (piece, time), accurate);
 333         if (!dec) {
 334                 return black_dcp_video (time);
 335         }
 336
 337         dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 338         if (_approximate_size) {
 339                 image_size.width &= ~3;
 340                 image_size.height &= ~3;
 341         }
 342
 343         shared_ptr<DCPVideo> dcp_video (
 344                 new DCPVideo (
 345                         dec->image,
 346                         dec->eyes,
 347                         content->crop (),
 348                         image_size,
 349                         _video_container_size,
 350                         _film->scaler(),
 351                         content->colour_conversion (),
 352                         time
 353                         )
 354                 );
 355
 356         /* Add subtitles */
 357
 358         ov = overlaps<SubtitleContent> (
 359                 time,
 360                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
 361                 );
 362
 363         list<PositionImage> sub_images;
 364
 365         for (list<shared_ptr<Piece> >::const_iterator i = ov.begin(); i != ov.end(); ++i) {
 366                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*i)->decoder);
 367                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*i)->content);
 368                 ContentTime const from = dcp_to_content_subtitle (*i, time);
 369                 ContentTime const to = from + ContentTime::from_frames (1, content->video_frame_rate ());
 370
 371                 list<shared_ptr<ContentImageSubtitle> > image_subtitles = subtitle_decoder->get_image_subtitles (from, to);
 372                 if (!image_subtitles.empty ()) {
 373                         list<PositionImage> im = process_content_image_subtitles (
 374                                 subtitle_content,
 375                                 image_subtitles
 376                                 );
 377
 378                         copy (im.begin(), im.end(), back_inserter (sub_images));
 379                 }
 380
 381                 if (_burn_subtitles) {
 382                         list<shared_ptr<ContentTextSubtitle> > text_subtitles = subtitle_decoder->get_text_subtitles (from, to);
 383                         if (!text_subtitles.empty ()) {
 384                                 list<PositionImage> im = process_content_text_subtitles (text_subtitles);
 385                                 copy (im.begin(), im.end(), back_inserter (sub_images));
 386                         }
 387                 }
 388         }
 389
 390         if (!sub_images.empty ()) {
 391                 dcp_video->set_subtitle (merge (sub_images));
 392         }
 393
 394         return dcp_video;
 395 }
 396
 397 shared_ptr<AudioBuffers>
 398 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
 399 {
 400         if (!_have_valid_pieces) {
 401                 setup_pieces ();
 402         }
 403
 404         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
 405
 406         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
 407         audio->make_silent ();
 408
 409         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
 410         if (ov.empty ()) {
 411                 return audio;
 412         }
 413
 414         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
 415
 416                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
 417                 assert (content);
 418                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 419                 assert (decoder);
 420
 421                 if (content->content_audio_frame_rate() == 0) {
 422                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
 423                          * audio stream).
 424                          */
 425                         continue;
 426                 }
 427
 428                 /* The time that we should request from the content */
 429                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
 430                 DCPTime offset;
 431                 if (request < DCPTime ()) {
 432                         /* We went off the start of the content, so we will need to offset
 433                            the stuff we get back.
 434                         */
 435                         offset = -request;
 436                         request = DCPTime ();
 437                 }
 438
 439                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
 440
 441                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
 442                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, length_frames, accurate);
 443
 444                 /* Gain */
 445                 if (content->audio_gain() != 0) {
 446                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
 447                         gain->apply_gain (content->audio_gain ());
 448                         all->audio = gain;
 449                 }
 450
 451                 /* Remap channels */
 452                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
 453                 dcp_mapped->make_silent ();
 454                 AudioMapping map = content->audio_mapping ();
 455                 for (int i = 0; i < map.content_channels(); ++i) {
 456                         for (int j = 0; j < _film->audio_channels(); ++j) {
 457                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 458                                         dcp_mapped->accumulate_channel (
 459                                                 all->audio.get(),
 460                                                 i,
 461                                                 j,
 462                                                 map.get (i, static_cast<dcp::Channel> (j))
 463                                                 );
 464                                 }
 465                         }
 466                 }
 467
 468                 all->audio = dcp_mapped;
 469
 470                 audio->accumulate_frames (
 471                         all->audio.get(),
 472                         content_frame - all->frame,
 473                         offset.frames (_film->audio_frame_rate()),
 474                         min (AudioFrame (all->audio->frames()), length_frames) - offset.frames (_film->audio_frame_rate ())
 475                         );
 476         }
 477
 478         return audio;
 479 }
 480
 481 VideoFrame
 482 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 483 {
 484         /* s is the offset of t from the start position of this content */
 485         DCPTime s = t - piece->content->position ();
 486         s = DCPTime (max (int64_t (0), s.get ()));
 487         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 488
 489         /* Convert this to the content frame */
 490         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) * piece->frc.factor ();
 491 }
 492
 493 AudioFrame
 494 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
 495 {
 496         /* s is the offset of t from the start position of this content */
 497         DCPTime s = t - piece->content->position ();
 498         s = DCPTime (max (int64_t (0), s.get ()));
 499         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 500
 501         /* Convert this to the content frame */
 502         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
 503 }
 504
 505 ContentTime
 506 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
 507 {
 508         /* s is the offset of t from the start position of this content */
 509         DCPTime s = t - piece->content->position ();
 510         s = DCPTime (max (int64_t (0), s.get ()));
 511         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 512
 513         return ContentTime (s, piece->frc);
 514 }
 515
 516 void
 517 PlayerStatistics::dump (shared_ptr<Log> log) const
 518 {
 519         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat));
 520         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()));
 521 }
 522
 523 PlayerStatistics const &
 524 Player::statistics () const
 525 {
 526         return _statistics;
 527 }