src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include <algorithm>
  22 #include "player.h"
  23 #include "film.h"
  24 #include "ffmpeg_decoder.h"
  25 #include "audio_buffers.h"
  26 #include "ffmpeg_content.h"
  27 #include "image_decoder.h"
  28 #include "image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "subrip_decoder.h"
  33 #include "subrip_content.h"
  34 #include "playlist.h"
  35 #include "job.h"
  36 #include "image.h"
  37 #include "ratio.h"
  38 #include "log.h"
  39 #include "scaler.h"
  40 #include "render_subtitles.h"
  41 #include "dcp_video.h"
  42 #include "config.h"
  43 #include "content_video.h"
  44
  45 using std::list;
  46 using std::cout;
  47 using std::min;
  48 using std::max;
  49 using std::min;
  50 using std::vector;
  51 using std::pair;
  52 using std::map;
  53 using std::make_pair;
  54 using boost::shared_ptr;
  55 using boost::weak_ptr;
  56 using boost::dynamic_pointer_cast;
  57 using boost::optional;
  58
  59 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  60         : _film (f)
  61         , _playlist (p)
  62         , _have_valid_pieces (false)
  63         , _approximate_size (false)
  64         , _burn_subtitles (false)
  65 {
  66         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  67         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  68         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  69         set_video_container_size (_film->frame_size ());
  70 }
  71
  72 void
  73 Player::setup_pieces ()
  74 {
  75         list<shared_ptr<Piece> > old_pieces = _pieces;
  76         _pieces.clear ();
  77
  78         ContentList content = _playlist->content ();
  79
  80         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
  81
  82                 if (!(*i)->paths_valid ()) {
  83                         continue;
  84                 }
  85
  86                 shared_ptr<Decoder> decoder;
  87                 optional<FrameRateChange> frc;
  88
  89                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
  90                 DCPTime best_overlap_t;
  91                 shared_ptr<VideoContent> best_overlap;
  92                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
  93                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
  94                         if (!vc) {
  95                                 continue;
  96                         }
  97
  98                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
  99                         if (overlap > best_overlap_t) {
 100                                 best_overlap = vc;
 101                                 best_overlap_t = overlap;
 102                         }
 103                 }
 104
 105                 optional<FrameRateChange> best_overlap_frc;
 106                 if (best_overlap) {
 107                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 108                 } else {
 109                         /* No video overlap; e.g. if the DCP is just audio */
 110                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 111                 }
 112
 113                 /* FFmpeg */
 114                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 115                 if (fc) {
 116                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
 117                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 118                 }
 119
 120                 /* ImageContent */
 121                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 122                 if (ic) {
 123                         /* See if we can re-use an old ImageDecoder */
 124                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 125                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 126                                 if (imd && imd->content() == ic) {
 127                                         decoder = imd;
 128                                 }
 129                         }
 130
 131                         if (!decoder) {
 132                                 decoder.reset (new ImageDecoder (ic));
 133                         }
 134
 135                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 136                 }
 137
 138                 /* SndfileContent */
 139                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 140                 if (sc) {
 141                         decoder.reset (new SndfileDecoder (sc));
 142                         frc = best_overlap_frc;
 143                 }
 144
 145                 /* SubRipContent */
 146                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
 147                 if (rc) {
 148                         decoder.reset (new SubRipDecoder (rc));
 149                         frc = best_overlap_frc;
 150                 }
 151
 152                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 153         }
 154
 155         _have_valid_pieces = true;
 156 }
 157
 158 void
 159 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 160 {
 161         shared_ptr<Content> c = w.lock ();
 162         if (!c) {
 163                 return;
 164         }
 165
 166         if (
 167                 property == ContentProperty::POSITION ||
 168                 property == ContentProperty::LENGTH ||
 169                 property == ContentProperty::TRIM_START ||
 170                 property == ContentProperty::TRIM_END ||
 171                 property == ContentProperty::PATH ||
 172                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 173                 ) {
 174
 175                 _have_valid_pieces = false;
 176                 Changed (frequent);
 177
 178         } else if (
 179                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 180                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 181                 property == SubtitleContentProperty::SUBTITLE_SCALE ||
 182                 property == VideoContentProperty::VIDEO_CROP ||
 183                 property == VideoContentProperty::VIDEO_SCALE ||
 184                 property == VideoContentProperty::VIDEO_FRAME_RATE
 185                 ) {
 186
 187                 Changed (frequent);
 188         }
 189 }
 190
 191 void
 192 Player::playlist_changed ()
 193 {
 194         _have_valid_pieces = false;
 195         Changed (false);
 196 }
 197
 198 void
 199 Player::set_video_container_size (dcp::Size s)
 200 {
 201         _video_container_size = s;
 202
 203         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 204         _black_image->make_black ();
 205 }
 206
 207 void
 208 Player::film_changed (Film::Property p)
 209 {
 210         /* Here we should notice Film properties that affect our output, and
 211            alert listeners that our output now would be different to how it was
 212            last time we were run.
 213         */
 214
 215         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 216                 Changed (false);
 217         }
 218 }
 219
 220 list<PositionImage>
 221 Player::process_content_image_subtitles (shared_ptr<SubtitleContent> content, list<shared_ptr<ContentImageSubtitle> > subs)
 222 {
 223         list<PositionImage> all;
 224
 225         for (list<shared_ptr<ContentImageSubtitle> >::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 226                 if (!(*i)->image) {
 227                         continue;
 228                 }
 229
 230                 dcpomatic::Rect<double> in_rect = (*i)->rectangle;
 231                 dcp::Size scaled_size;
 232
 233                 in_rect.x += content->subtitle_x_offset ();
 234                 in_rect.y += content->subtitle_y_offset ();
 235
 236                 /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 237                 scaled_size.width = in_rect.width * _video_container_size.width * content->subtitle_scale ();
 238                 scaled_size.height = in_rect.height * _video_container_size.height * content->subtitle_scale ();
 239
 240                 /* Then we need a corrective translation, consisting of two parts:
 241                  *
 242                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 243                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 244                  *
 245                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 246                  *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 247                  *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 248                  *
 249                  * Combining these two translations gives these expressions.
 250                  */
 251
 252                 all.push_back (
 253                         PositionImage (
 254                                 (*i)->image->scale (
 255                                         scaled_size,
 256                                         Scaler::from_id ("bicubic"),
 257                                         (*i)->image->pixel_format (),
 258                                         true
 259                                         ),
 260                                 Position<int> (
 261                                         rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - content->subtitle_scale ()) / 2))),
 262                                         rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - content->subtitle_scale ()) / 2)))
 263                                         )
 264                                 )
 265                         );
 266         }
 267
 268         return all;
 269 }
 270
 271 list<PositionImage>
 272 Player::process_content_text_subtitles (list<shared_ptr<ContentTextSubtitle> > sub)
 273 {
 274         list<PositionImage> all;
 275         for (list<shared_ptr<ContentTextSubtitle> >::const_iterator i = sub.begin(); i != sub.end(); ++i) {
 276                 if (!(*i)->subs.empty ()) {
 277                         all.push_back (render_subtitles ((*i)->subs, _video_container_size));
 278                 }
 279         }
 280
 281         return all;
 282 }
 283
 284 void
 285 Player::set_approximate_size ()
 286 {
 287         _approximate_size = true;
 288 }
 289
 290 shared_ptr<DCPVideo>
 291 Player::get_video (DCPTime time, bool accurate)
 292 {
 293         if (!_have_valid_pieces) {
 294                 setup_pieces ();
 295         }
 296
 297         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (time);
 298         if (ov.empty ()) {
 299                 /* No video content at this time: return a black frame */
 300                 return shared_ptr<DCPVideo> (
 301                         new DCPVideo (
 302                                 _black_image,
 303                                 EYES_BOTH,
 304                                 Crop (),
 305                                 _video_container_size,
 306                                 _video_container_size,
 307                                 Scaler::from_id ("bicubic"),
 308                                 Config::instance()->colour_conversions().front().conversion,
 309                                 time
 310                                 )
 311                         );
 312         }
 313
 314         /* Create a DCPVideo from the content's video at this time */
 315
 316         shared_ptr<Piece> piece = ov.back ();
 317         shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
 318         assert (decoder);
 319         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 320         assert (content);
 321
 322         optional<ContentVideo> dec = decoder->get_video (dcp_to_content_video (piece, time), accurate);
 323         assert (dec);
 324
 325         dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 326         if (_approximate_size) {
 327                 image_size.width &= ~3;
 328                 image_size.height &= ~3;
 329         }
 330
 331         shared_ptr<DCPVideo> dcp_video (
 332                 new DCPVideo (
 333                         dec->image,
 334                         dec->eyes,
 335                         content->crop (),
 336                         image_size,
 337                         _video_container_size,
 338                         _film->scaler(),
 339                         content->colour_conversion (),
 340                         time
 341                         )
 342                 );
 343
 344         /* Add subtitles */
 345
 346         ov = overlaps<SubtitleContent> (time);
 347         list<PositionImage> sub_images;
 348
 349         for (list<shared_ptr<Piece> >::const_iterator i = ov.begin(); i != ov.end(); ++i) {
 350                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*i)->decoder);
 351                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*i)->content);
 352                 ContentTime const from = dcp_to_content_subtitle (*i, time);
 353                 ContentTime const to = from + ContentTime::from_frames (1, content->video_frame_rate ());
 354
 355                 list<shared_ptr<ContentImageSubtitle> > image_subtitles = subtitle_decoder->get_image_subtitles (from, to);
 356                 if (!image_subtitles.empty ()) {
 357                         list<PositionImage> im = process_content_image_subtitles (
 358                                 subtitle_content,
 359                                 image_subtitles
 360                                 );
 361
 362                         copy (im.begin(), im.end(), back_inserter (sub_images));
 363                 }
 364
 365                 if (_burn_subtitles) {
 366                         list<shared_ptr<ContentTextSubtitle> > text_subtitles = subtitle_decoder->get_text_subtitles (from, to);
 367                         if (!text_subtitles.empty ()) {
 368                                 list<PositionImage> im = process_content_text_subtitles (text_subtitles);
 369                                 copy (im.begin(), im.end(), back_inserter (sub_images));
 370                         }
 371                 }
 372         }
 373
 374         if (!sub_images.empty ()) {
 375                 dcp_video->set_subtitle (merge (sub_images));
 376         }
 377
 378         return dcp_video;
 379 }
 380
 381 shared_ptr<AudioBuffers>
 382 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
 383 {
 384         if (!_have_valid_pieces) {
 385                 setup_pieces ();
 386         }
 387
 388         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
 389
 390         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
 391         audio->make_silent ();
 392
 393         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time);
 394         if (ov.empty ()) {
 395                 return audio;
 396         }
 397
 398         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
 399
 400                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
 401                 assert (content);
 402                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 403                 assert (decoder);
 404
 405                 AudioFrame const content_time = dcp_to_content_audio (*i, time);
 406
 407                 /* Audio from this piece's decoder (which might be more than what we asked for) */
 408                 shared_ptr<ContentAudio> all = decoder->get_audio (content_time, length_frames, accurate);
 409
 410                 /* Gain */
 411                 if (content->audio_gain() != 0) {
 412                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
 413                         gain->apply_gain (content->audio_gain ());
 414                         all->audio = gain;
 415                 }
 416
 417                 /* Remap channels */
 418                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
 419                 dcp_mapped->make_silent ();
 420                 AudioMapping map = content->audio_mapping ();
 421                 for (int i = 0; i < map.content_channels(); ++i) {
 422                         for (int j = 0; j < _film->audio_channels(); ++j) {
 423                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 424                                         dcp_mapped->accumulate_channel (
 425                                                 all->audio.get(),
 426                                                 i,
 427                                                 j,
 428                                                 map.get (i, static_cast<dcp::Channel> (j))
 429                                                 );
 430                                 }
 431                         }
 432                 }
 433
 434                 all->audio = dcp_mapped;
 435
 436                 /* Delay */
 437                 /* XXX
 438                 audio->dcp_time += content->audio_delay() * TIME_HZ / 1000;
 439                 if (audio->dcp_time < 0) {
 440                         int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ;
 441                         if (frames >= audio->audio->frames ()) {
 442                                 return;
 443                         }
 444
 445                         shared_ptr<AudioBuffers> trimmed (new AudioBuffers (audio->audio->channels(), audio->audio->frames() - frames));
 446                         trimmed->copy_from (audio->audio.get(), audio->audio->frames() - frames, frames, 0);
 447
 448                         audio->audio = trimmed;
 449                         audio->dcp_time = 0;
 450                 }
 451                 */
 452
 453                 audio->accumulate_frames (all->audio.get(), all->frame - content_time, 0, min (AudioFrame (all->audio->frames()), length_frames));
 454         }
 455
 456         return audio;
 457 }
 458
 459 VideoFrame
 460 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 461 {
 462         /* s is the offset of t from the start position of this content */
 463         DCPTime s = t - piece->content->position ();
 464         s = DCPTime (max (int64_t (0), s.get ()));
 465         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 466
 467         /* Convert this to the content frame */
 468         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) * piece->frc.factor ();
 469 }
 470
 471 AudioFrame
 472 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
 473 {
 474         /* s is the offset of t from the start position of this content */
 475         DCPTime s = t - piece->content->position ();
 476         s = DCPTime (max (int64_t (0), s.get ()));
 477         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 478
 479         /* Convert this to the content frame */
 480         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
 481 }
 482
 483 ContentTime
 484 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
 485 {
 486         /* s is the offset of t from the start position of this content */
 487         DCPTime s = t - piece->content->position ();
 488         s = DCPTime (max (int64_t (0), s.get ()));
 489         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 490
 491         return ContentTime (s, piece->frc);
 492 }
 493
 494 void
 495 PlayerStatistics::dump (shared_ptr<Log> log) const
 496 {
 497         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat));
 498         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()));
 499 }
 500
 501 PlayerStatistics const &
 502 Player::statistics () const
 503 {
 504         return _statistics;
 505 }