src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include <algorithm>
  22 #include "player.h"
  23 #include "film.h"
  24 #include "ffmpeg_decoder.h"
  25 #include "audio_buffers.h"
  26 #include "ffmpeg_content.h"
  27 #include "image_decoder.h"
  28 #include "image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "subrip_decoder.h"
  33 #include "subrip_content.h"
  34 #include "playlist.h"
  35 #include "job.h"
  36 #include "image.h"
  37 #include "ratio.h"
  38 #include "log.h"
  39 #include "scaler.h"
  40 #include "render_subtitles.h"
  41 #include "dcp_video.h"
  42 #include "config.h"
  43 #include "content_video.h"
  44
  45 using std::list;
  46 using std::cout;
  47 using std::min;
  48 using std::max;
  49 using std::min;
  50 using std::vector;
  51 using std::pair;
  52 using std::map;
  53 using std::make_pair;
  54 using boost::shared_ptr;
  55 using boost::weak_ptr;
  56 using boost::dynamic_pointer_cast;
  57 using boost::optional;
  58
  59 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  60         : _film (f)
  61         , _playlist (p)
  62         , _have_valid_pieces (false)
  63         , _approximate_size (false)
  64         , _burn_subtitles (false)
  65 {
  66         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  67         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  68         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  69         set_video_container_size (_film->frame_size ());
  70 }
  71
  72 void
  73 Player::setup_pieces ()
  74 {
  75         list<shared_ptr<Piece> > old_pieces = _pieces;
  76         _pieces.clear ();
  77
  78         ContentList content = _playlist->content ();
  79
  80         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
  81
  82                 if (!(*i)->paths_valid ()) {
  83                         continue;
  84                 }
  85
  86                 shared_ptr<Decoder> decoder;
  87                 optional<FrameRateChange> frc;
  88
  89                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
  90                 DCPTime best_overlap_t;
  91                 shared_ptr<VideoContent> best_overlap;
  92                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
  93                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
  94                         if (!vc) {
  95                                 continue;
  96                         }
  97
  98                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
  99                         if (overlap > best_overlap_t) {
 100                                 best_overlap = vc;
 101                                 best_overlap_t = overlap;
 102                         }
 103                 }
 104
 105                 optional<FrameRateChange> best_overlap_frc;
 106                 if (best_overlap) {
 107                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 108                 } else {
 109                         /* No video overlap; e.g. if the DCP is just audio */
 110                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 111                 }
 112
 113                 /* FFmpeg */
 114                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 115                 if (fc) {
 116                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
 117                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 118                 }
 119
 120                 /* ImageContent */
 121                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 122                 if (ic) {
 123                         /* See if we can re-use an old ImageDecoder */
 124                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 125                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 126                                 if (imd && imd->content() == ic) {
 127                                         decoder = imd;
 128                                 }
 129                         }
 130
 131                         if (!decoder) {
 132                                 decoder.reset (new ImageDecoder (ic));
 133                         }
 134
 135                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 136                 }
 137
 138                 /* SndfileContent */
 139                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 140                 if (sc) {
 141                         decoder.reset (new SndfileDecoder (sc));
 142                         frc = best_overlap_frc;
 143                 }
 144
 145                 /* SubRipContent */
 146                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
 147                 if (rc) {
 148                         decoder.reset (new SubRipDecoder (rc));
 149                         frc = best_overlap_frc;
 150                 }
 151
 152                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 153         }
 154
 155         _have_valid_pieces = true;
 156 }
 157
 158 void
 159 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 160 {
 161         shared_ptr<Content> c = w.lock ();
 162         if (!c) {
 163                 return;
 164         }
 165
 166         if (
 167                 property == ContentProperty::POSITION ||
 168                 property == ContentProperty::LENGTH ||
 169                 property == ContentProperty::TRIM_START ||
 170                 property == ContentProperty::TRIM_END ||
 171                 property == ContentProperty::PATH ||
 172                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 173                 ) {
 174
 175                 _have_valid_pieces = false;
 176                 Changed (frequent);
 177
 178         } else if (
 179                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 180                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 181                 property == SubtitleContentProperty::SUBTITLE_SCALE ||
 182                 property == VideoContentProperty::VIDEO_CROP ||
 183                 property == VideoContentProperty::VIDEO_SCALE ||
 184                 property == VideoContentProperty::VIDEO_FRAME_RATE
 185                 ) {
 186
 187                 Changed (frequent);
 188         }
 189 }
 190
 191 void
 192 Player::playlist_changed ()
 193 {
 194         _have_valid_pieces = false;
 195         Changed (false);
 196 }
 197
 198 void
 199 Player::set_video_container_size (dcp::Size s)
 200 {
 201         _video_container_size = s;
 202
 203         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 204         _black_image->make_black ();
 205 }
 206
 207 void
 208 Player::film_changed (Film::Property p)
 209 {
 210         /* Here we should notice Film properties that affect our output, and
 211            alert listeners that our output now would be different to how it was
 212            last time we were run.
 213         */
 214
 215         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 216                 Changed (false);
 217         }
 218 }
 219
 220 list<PositionImage>
 221 Player::process_content_image_subtitles (shared_ptr<SubtitleContent> content, list<shared_ptr<ContentImageSubtitle> > subs)
 222 {
 223         list<PositionImage> all;
 224
 225         for (list<shared_ptr<ContentImageSubtitle> >::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 226                 if (!(*i)->image) {
 227                         continue;
 228                 }
 229
 230                 dcpomatic::Rect<double> in_rect = (*i)->rectangle;
 231                 dcp::Size scaled_size;
 232
 233                 in_rect.x += content->subtitle_x_offset ();
 234                 in_rect.y += content->subtitle_y_offset ();
 235
 236                 /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 237                 scaled_size.width = in_rect.width * _video_container_size.width * content->subtitle_scale ();
 238                 scaled_size.height = in_rect.height * _video_container_size.height * content->subtitle_scale ();
 239
 240                 /* Then we need a corrective translation, consisting of two parts:
 241                  *
 242                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 243                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 244                  *
 245                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 246                  *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 247                  *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 248                  *
 249                  * Combining these two translations gives these expressions.
 250                  */
 251
 252                 all.push_back (
 253                         PositionImage (
 254                                 (*i)->image->scale (
 255                                         scaled_size,
 256                                         Scaler::from_id ("bicubic"),
 257                                         (*i)->image->pixel_format (),
 258                                         true
 259                                         ),
 260                                 Position<int> (
 261                                         rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - content->subtitle_scale ()) / 2))),
 262                                         rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - content->subtitle_scale ()) / 2)))
 263                                         )
 264                                 )
 265                         );
 266         }
 267
 268         return all;
 269 }
 270
 271 list<PositionImage>
 272 Player::process_content_text_subtitles (list<shared_ptr<ContentTextSubtitle> > sub)
 273 {
 274         list<PositionImage> all;
 275         for (list<shared_ptr<ContentTextSubtitle> >::const_iterator i = sub.begin(); i != sub.end(); ++i) {
 276                 if (!(*i)->subs.empty ()) {
 277                         all.push_back (render_subtitles ((*i)->subs, _video_container_size));
 278                 }
 279         }
 280
 281         return all;
 282 }
 283
 284 void
 285 Player::set_approximate_size ()
 286 {
 287         _approximate_size = true;
 288 }
 289
 290 shared_ptr<DCPVideo>
 291 Player::get_video (DCPTime time, bool accurate)
 292 {
 293         if (!_have_valid_pieces) {
 294                 setup_pieces ();
 295         }
 296
 297         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (time);
 298         if (ov.empty ()) {
 299                 /* No video content at this time: return a black frame */
 300                 return shared_ptr<DCPVideo> (
 301                         new DCPVideo (
 302                                 _black_image,
 303                                 EYES_BOTH,
 304                                 Crop (),
 305                                 _video_container_size,
 306                                 _video_container_size,
 307                                 Scaler::from_id ("bicubic"),
 308                                 Config::instance()->colour_conversions().front().conversion,
 309                                 time
 310                                 )
 311                         );
 312         }
 313
 314         /* Create a DCPVideo from the content's video at this time */
 315
 316         shared_ptr<Piece> piece = ov.back ();
 317         shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
 318         assert (decoder);
 319         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 320         assert (content);
 321
 322         shared_ptr<ContentVideo> dec = decoder->get_video (dcp_to_content_video (piece, time), accurate);
 323
 324         dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 325         if (_approximate_size) {
 326                 image_size.width &= ~3;
 327                 image_size.height &= ~3;
 328         }
 329
 330         shared_ptr<DCPVideo> dcp_video (
 331                 new DCPVideo (
 332                         dec->image,
 333                         dec->eyes,
 334                         content->crop (),
 335                         image_size,
 336                         _video_container_size,
 337                         _film->scaler(),
 338                         content->colour_conversion (),
 339                         time
 340                         )
 341                 );
 342
 343         /* Add subtitles */
 344
 345         ov = overlaps<SubtitleContent> (time);
 346         list<PositionImage> sub_images;
 347
 348         for (list<shared_ptr<Piece> >::const_iterator i = ov.begin(); i != ov.end(); ++i) {
 349                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*i)->decoder);
 350                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*i)->content);
 351                 ContentTime const from = dcp_to_content_subtitle (*i, time);
 352                 ContentTime const to = from + ContentTime::from_frames (1, content->video_frame_rate ());
 353
 354                 list<shared_ptr<ContentImageSubtitle> > image_subtitles = subtitle_decoder->get_image_subtitles (from, to);
 355                 if (!image_subtitles.empty ()) {
 356                         list<PositionImage> im = process_content_image_subtitles (
 357                                 subtitle_content,
 358                                 image_subtitles
 359                                 );
 360
 361                         copy (im.begin(), im.end(), back_inserter (sub_images));
 362                 }
 363
 364                 if (_burn_subtitles) {
 365                         list<shared_ptr<ContentTextSubtitle> > text_subtitles = subtitle_decoder->get_text_subtitles (from, to);
 366                         if (!text_subtitles.empty ()) {
 367                                 list<PositionImage> im = process_content_text_subtitles (text_subtitles);
 368                                 copy (im.begin(), im.end(), back_inserter (sub_images));
 369                         }
 370                 }
 371         }
 372
 373         if (!sub_images.empty ()) {
 374                 dcp_video->set_subtitle (merge (sub_images));
 375         }
 376
 377         return dcp_video;
 378 }
 379
 380 shared_ptr<AudioBuffers>
 381 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
 382 {
 383         if (!_have_valid_pieces) {
 384                 setup_pieces ();
 385         }
 386
 387         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
 388
 389         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
 390         audio->make_silent ();
 391
 392         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time);
 393         if (ov.empty ()) {
 394                 return audio;
 395         }
 396
 397         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
 398
 399                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
 400                 assert (content);
 401                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 402                 assert (decoder);
 403
 404                 AudioFrame const content_time = dcp_to_content_audio (*i, time);
 405
 406                 /* Audio from this piece's decoder (which might be more than what we asked for) */
 407                 shared_ptr<ContentAudio> all = decoder->get_audio (content_time, length_frames, accurate);
 408
 409                 /* Gain */
 410                 if (content->audio_gain() != 0) {
 411                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
 412                         gain->apply_gain (content->audio_gain ());
 413                         all->audio = gain;
 414                 }
 415
 416                 /* Remap channels */
 417                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
 418                 dcp_mapped->make_silent ();
 419                 AudioMapping map = content->audio_mapping ();
 420                 for (int i = 0; i < map.content_channels(); ++i) {
 421                         for (int j = 0; j < _film->audio_channels(); ++j) {
 422                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 423                                         dcp_mapped->accumulate_channel (
 424                                                 all->audio.get(),
 425                                                 i,
 426                                                 j,
 427                                                 map.get (i, static_cast<dcp::Channel> (j))
 428                                                 );
 429                                 }
 430                         }
 431                 }
 432
 433                 all->audio = dcp_mapped;
 434
 435                 /* Delay */
 436                 /* XXX
 437                 audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
 438                 if (audio->dcp_time < 0) {
 439                         int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
 440                         if (frames >= audio->audio->frames ()) {
 441                                 return;
 442                         }
 443
 444                         shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->audio->channels(), audio->audio->frames() - frames));
 445                         trimmed->copy_from (audio->audio.get(), audio->audio->frames() - frames, frames, 0);
 446
 447                         audio->audio = trimmed;
 448                         audio->dcp_time = 0;
 449                 }
 450                 */
 451
 452                 audio->accumulate_frames (all->audio.get(), all->frame - content_time, 0, min (AudioFrame (all->audio->frames()), length_frames));
 453         }
 454
 455         return audio;
 456 }
 457
 458 VideoFrame
 459 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 460 {
 461         /* s is the offset of t from the start position of this content */
 462         DCPTime s = t - piece->content->position ();
 463         s = DCPTime (max (int64_t (0), s.get ()));
 464         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 465
 466         /* Convert this to the content frame */
 467         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) * piece->frc.factor ();
 468 }
 469
 470 AudioFrame
 471 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
 472 {
 473         /* s is the offset of t from the start position of this content */
 474         DCPTime s = t - piece->content->position ();
 475         s = DCPTime (max (int64_t (0), s.get ()));
 476         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 477
 478         /* Convert this to the content frame */
 479         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
 480 }
 481
 482 ContentTime
 483 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
 484 {
 485         /* s is the offset of t from the start position of this content */
 486         DCPTime s = t - piece->content->position ();
 487         s = DCPTime (max (int64_t (0), s.get ()));
 488         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 489
 490         return ContentTime (s, piece->frc);
 491 }
 492
 493 void
 494 PlayerStatistics::dump (shared_ptr<Log> log) const
 495 {
 496         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat));
 497         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()));
 498 }
 499
 500 PlayerStatistics const &
 501 Player::statistics () const
 502 {
 503         return _statistics;
 504 }