src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include <algorithm>
  22 #include "player.h"
  23 #include "film.h"
  24 #include "ffmpeg_decoder.h"
  25 #include "audio_buffers.h"
  26 #include "ffmpeg_content.h"
  27 #include "image_decoder.h"
  28 #include "image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "subrip_decoder.h"
  33 #include "subrip_content.h"
  34 #include "dcp_content.h"
  35 #include "playlist.h"
  36 #include "job.h"
  37 #include "image.h"
  38 #include "raw_image_proxy.h"
  39 #include "ratio.h"
  40 #include "log.h"
  41 #include "scaler.h"
  42 #include "render_subtitles.h"
  43 #include "config.h"
  44 #include "content_video.h"
  45 #include "player_video.h"
  46 #include "frame_rate_change.h"
  47 #include "dcp_content.h"
  48 #include "dcp_decoder.h"
  49 #include "dcp_subtitle_content.h"
  50 #include "dcp_subtitle_decoder.h"
  51
  52 #define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
  53
  54 using std::list;
  55 using std::cout;
  56 using std::min;
  57 using std::max;
  58 using std::min;
  59 using std::vector;
  60 using std::pair;
  61 using std::map;
  62 using std::make_pair;
  63 using boost::shared_ptr;
  64 using boost::weak_ptr;
  65 using boost::dynamic_pointer_cast;
  66 using boost::optional;
  67
  68 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  69         : _film (f)
  70         , _playlist (p)
  71         , _have_valid_pieces (false)
  72         , _approximate_size (false)
  73 {
  74         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  75         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  76         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  77         set_video_container_size (_film->frame_size ());
  78 }
  79
  80 void
  81 Player::setup_pieces ()
  82 {
  83         list<shared_ptr<Piece> > old_pieces = _pieces;
  84         _pieces.clear ();
  85
  86         ContentList content = _playlist->content ();
  87
  88         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
  89
  90                 if (!(*i)->paths_valid ()) {
  91                         continue;
  92                 }
  93
  94                 shared_ptr<Decoder> decoder;
  95                 optional<FrameRateChange> frc;
  96
  97                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
  98                 DCPTime best_overlap_t;
  99                 shared_ptr<VideoContent> best_overlap;
 100                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
 101                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
 102                         if (!vc) {
 103                                 continue;
 104                         }
 105
 106                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
 107                         if (overlap > best_overlap_t) {
 108                                 best_overlap = vc;
 109                                 best_overlap_t = overlap;
 110                         }
 111                 }
 112
 113                 optional<FrameRateChange> best_overlap_frc;
 114                 if (best_overlap) {
 115                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 116                 } else {
 117                         /* No video overlap; e.g. if the DCP is just audio */
 118                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 119                 }
 120
 121                 /* FFmpeg */
 122                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 123                 if (fc) {
 124                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
 125                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 126                 }
 127
 128                 shared_ptr<const DCPContent> dc = dynamic_pointer_cast<const DCPContent> (*i);
 129                 if (dc) {
 130                         decoder.reset (new DCPDecoder (dc, _film->log ()));
 131                         frc = FrameRateChange (dc->video_frame_rate(), _film->video_frame_rate());
 132                 }
 133
 134                 /* ImageContent */
 135                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 136                 if (ic) {
 137                         /* See if we can re-use an old ImageDecoder */
 138                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 139                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 140                                 if (imd && imd->content() == ic) {
 141                                         decoder = imd;
 142                                 }
 143                         }
 144
 145                         if (!decoder) {
 146                                 decoder.reset (new ImageDecoder (ic));
 147                         }
 148
 149                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 150                 }
 151
 152                 /* SndfileContent */
 153                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 154                 if (sc) {
 155                         decoder.reset (new SndfileDecoder (sc));
 156                         frc = best_overlap_frc;
 157                 }
 158
 159                 /* SubRipContent */
 160                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
 161                 if (rc) {
 162                         decoder.reset (new SubRipDecoder (rc));
 163                         frc = best_overlap_frc;
 164                 }
 165
 166                 /* DCPSubtitleContent */
 167                 shared_ptr<const DCPSubtitleContent> dsc = dynamic_pointer_cast<const DCPSubtitleContent> (*i);
 168                 if (dsc) {
 169                         decoder.reset (new DCPSubtitleDecoder (dsc));
 170                         frc = best_overlap_frc;
 171                 }
 172
 173                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 174         }
 175
 176         _have_valid_pieces = true;
 177 }
 178
 179 void
 180 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 181 {
 182         shared_ptr<Content> c = w.lock ();
 183         if (!c) {
 184                 return;
 185         }
 186
 187         if (
 188                 property == ContentProperty::POSITION ||
 189                 property == ContentProperty::LENGTH ||
 190                 property == ContentProperty::TRIM_START ||
 191                 property == ContentProperty::TRIM_END ||
 192                 property == ContentProperty::PATH ||
 193                 property == VideoContentProperty::VIDEO_FRAME_TYPE ||
 194                 property == DCPContentProperty::CAN_BE_PLAYED
 195                 ) {
 196
 197                 _have_valid_pieces = false;
 198                 Changed (frequent);
 199
 200         } else if (
 201                 property == SubtitleContentProperty::USE_SUBTITLES ||
 202                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 203                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 204                 property == SubtitleContentProperty::SUBTITLE_SCALE ||
 205                 property == VideoContentProperty::VIDEO_CROP ||
 206                 property == VideoContentProperty::VIDEO_SCALE ||
 207                 property == VideoContentProperty::VIDEO_FRAME_RATE
 208                 ) {
 209
 210                 Changed (frequent);
 211         }
 212 }
 213
 214 /** @param already_resampled true if this data has already been through the chain up to the resampler */
 215 void
 216 Player::playlist_changed ()
 217 {
 218         _have_valid_pieces = false;
 219         Changed (false);
 220 }
 221
 222 void
 223 Player::set_video_container_size (dcp::Size s)
 224 {
 225         _video_container_size = s;
 226
 227         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 228         _black_image->make_black ();
 229 }
 230
 231 void
 232 Player::film_changed (Film::Property p)
 233 {
 234         /* Here we should notice Film properties that affect our output, and
 235            alert listeners that our output now would be different to how it was
 236            last time we were run.
 237         */
 238
 239         if (p == Film::SCALER || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 240                 Changed (false);
 241         }
 242 }
 243
 244 list<PositionImage>
 245 Player::transform_image_subtitles (list<ImageSubtitle> subs) const
 246 {
 247         list<PositionImage> all;
 248
 249         for (list<ImageSubtitle>::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 250                 if (!i->image) {
 251                         continue;
 252                 }
 253
 254                 /* We will scale the subtitle up to fit _video_container_size */
 255                 dcp::Size scaled_size (i->rectangle.width * _video_container_size.width, i->rectangle.height * _video_container_size.height);
 256
 257                 /* Then we need a corrective translation, consisting of two parts:
 258                  *
 259                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 260                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 261                  *
 262                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 263                  *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 264                  *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 265                  *
 266                  * Combining these two translations gives these expressions.
 267                  */
 268
 269                 all.push_back (
 270                         PositionImage (
 271                                 i->image->scale (
 272                                         scaled_size,
 273                                         Scaler::from_id ("bicubic"),
 274                                         i->image->pixel_format (),
 275                                         true
 276                                         ),
 277                                 Position<int> (
 278                                         rint (_video_container_size.width * i->rectangle.x),
 279                                         rint (_video_container_size.height * i->rectangle.y)
 280                                         )
 281                                 )
 282                         );
 283         }
 284
 285         return all;
 286 }
 287
 288 void
 289 Player::set_approximate_size ()
 290 {
 291         _approximate_size = true;
 292 }
 293
 294 shared_ptr<PlayerVideo>
 295 Player::black_player_video_frame (DCPTime time) const
 296 {
 297         return shared_ptr<PlayerVideo> (
 298                 new PlayerVideo (
 299                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image, _film->log ())),
 300                         time,
 301                         Crop (),
 302                         _video_container_size,
 303                         _video_container_size,
 304                         Scaler::from_id ("bicubic"),
 305                         EYES_BOTH,
 306                         PART_WHOLE,
 307                         Config::instance()->colour_conversions().front().conversion
 308                 )
 309         );
 310 }
 311
 312 /** @return All PlayerVideos at the given time (there may be two frames for 3D) */
 313 list<shared_ptr<PlayerVideo> >
 314 Player::get_video (DCPTime time, bool accurate)
 315 {
 316         if (!_have_valid_pieces) {
 317                 setup_pieces ();
 318         }
 319
 320         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
 321                 time,
 322                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
 323                 );
 324
 325         list<shared_ptr<PlayerVideo> > pvf;
 326
 327         if (ov.empty ()) {
 328                 /* No video content at this time */
 329                 pvf.push_back (black_player_video_frame (time));
 330         } else {
 331                 /* Create a PlayerVideo from the content's video at this time */
 332
 333                 shared_ptr<Piece> piece = ov.back ();
 334                 shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
 335                 assert (decoder);
 336                 shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 337                 assert (content);
 338
 339                 list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
 340                 if (content_video.empty ()) {
 341                         pvf.push_back (black_player_video_frame (time));
 342                         return pvf;
 343                 }
 344
 345                 dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size (), _approximate_size ? 4 : 1);
 346                 if (_approximate_size) {
 347                         image_size.width &= ~3;
 348                         image_size.height &= ~3;
 349                 }
 350
 351                 for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
 352                         pvf.push_back (
 353                                 shared_ptr<PlayerVideo> (
 354                                         new PlayerVideo (
 355                                                 i->image,
 356                                                 content_video_to_dcp (piece, i->frame),
 357                                                 content->crop (),
 358                                                 image_size,
 359                                                 _video_container_size,
 360                                                 _film->scaler(),
 361                                                 i->eyes,
 362                                                 i->part,
 363                                                 content->colour_conversion ()
 364                                                 )
 365                                         )
 366                                 );
 367                 }
 368         }
 369
 370         /* Add subtitles (for possible burn-in) to whatever PlayerVideos we got */
 371
 372         PlayerSubtitles ps = get_subtitles (time, DCPTime::from_frames (1, _film->video_frame_rate ()), false);
 373
 374         list<PositionImage> sub_images;
 375
 376         /* Image subtitles */
 377         list<PositionImage> c = transform_image_subtitles (ps.image);
 378         copy (c.begin(), c.end(), back_inserter (sub_images));
 379
 380         /* Text subtitles (rendered to images) */
 381         sub_images.push_back (render_subtitles (ps.text, _video_container_size));
 382
 383         if (!sub_images.empty ()) {
 384                 for (list<shared_ptr<PlayerVideo> >::const_iterator i = pvf.begin(); i != pvf.end(); ++i) {
 385                         (*i)->set_subtitle (merge (sub_images));
 386                 }
 387         }
 388
 389         return pvf;
 390 }
 391
 392 shared_ptr<AudioBuffers>
 393 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
 394 {
 395         if (!_have_valid_pieces) {
 396                 setup_pieces ();
 397         }
 398
 399         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
 400
 401         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
 402         audio->make_silent ();
 403
 404         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
 405         if (ov.empty ()) {
 406                 return audio;
 407         }
 408
 409         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
 410
 411                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
 412                 assert (content);
 413                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 414                 assert (decoder);
 415
 416                 if (content->audio_frame_rate() == 0) {
 417                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
 418                          * audio stream).
 419                          */
 420                         continue;
 421                 }
 422
 423                 /* The time that we should request from the content */
 424                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
 425                 DCPTime offset;
 426                 if (request < DCPTime ()) {
 427                         /* We went off the start of the content, so we will need to offset
 428                            the stuff we get back.
 429                         */
 430                         offset = -request;
 431                         request = DCPTime ();
 432                 }
 433
 434                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
 435
 436                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
 437                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, length_frames, accurate);
 438
 439                 /* Gain */
 440                 if (content->audio_gain() != 0) {
 441                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
 442                         gain->apply_gain (content->audio_gain ());
 443                         all->audio = gain;
 444                 }
 445
 446                 /* Remap channels */
 447                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
 448                 dcp_mapped->make_silent ();
 449                 AudioMapping map = content->audio_mapping ();
 450                 for (int i = 0; i < map.content_channels(); ++i) {
 451                         for (int j = 0; j < _film->audio_channels(); ++j) {
 452                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 453                                         dcp_mapped->accumulate_channel (
 454                                                 all->audio.get(),
 455                                                 i,
 456                                                 j,
 457                                                 map.get (i, static_cast<dcp::Channel> (j))
 458                                                 );
 459                                 }
 460                         }
 461                 }
 462
 463                 all->audio = dcp_mapped;
 464
 465                 audio->accumulate_frames (
 466                         all->audio.get(),
 467                         content_frame - all->frame,
 468                         offset.frames (_film->audio_frame_rate()),
 469                         min (AudioFrame (all->audio->frames()), length_frames) - offset.frames (_film->audio_frame_rate ())
 470                         );
 471         }
 472
 473         return audio;
 474 }
 475
 476 VideoFrame
 477 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 478 {
 479         /* s is the offset of t from the start position of this content */
 480         DCPTime s = t - piece->content->position ();
 481         s = DCPTime (max (DCPTime::Type (0), s.get ()));
 482         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 483
 484         /* Convert this to the content frame */
 485         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) * piece->frc.factor ();
 486 }
 487
 488 DCPTime
 489 Player::content_video_to_dcp (shared_ptr<const Piece> piece, VideoFrame f) const
 490 {
 491         DCPTime t = DCPTime::from_frames (f / piece->frc.factor (), _film->video_frame_rate()) - piece->content->trim_start () + piece->content->position ();
 492         if (t < DCPTime ()) {
 493                 t = DCPTime ();
 494         }
 495
 496         return t;
 497 }
 498
 499 AudioFrame
 500 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
 501 {
 502         /* s is the offset of t from the start position of this content */
 503         DCPTime s = t - piece->content->position ();
 504         s = DCPTime (max (DCPTime::Type (0), s.get ()));
 505         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 506
 507         /* Convert this to the content frame */
 508         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
 509 }
 510
 511 ContentTime
 512 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
 513 {
 514         /* s is the offset of t from the start position of this content */
 515         DCPTime s = t - piece->content->position ();
 516         s = DCPTime (max (DCPTime::Type (0), s.get ()));
 517         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 518
 519         return ContentTime (s + piece->content->trim_start(), piece->frc);
 520 }
 521
 522 void
 523 PlayerStatistics::dump (shared_ptr<Log> log) const
 524 {
 525         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat), Log::TYPE_GENERAL);
 526         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()), Log::TYPE_GENERAL);
 527 }
 528
 529 PlayerStatistics const &
 530 Player::statistics () const
 531 {
 532         return _statistics;
 533 }
 534
 535 PlayerSubtitles
 536 Player::get_subtitles (DCPTime time, DCPTime length, bool starting)
 537 {
 538         list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (time, time + length);
 539
 540         PlayerSubtitles ps (time, length);
 541
 542         for (list<shared_ptr<Piece> >::const_iterator j = subs.begin(); j != subs.end(); ++j) {
 543                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*j)->content);
 544                 if (!subtitle_content->use_subtitles ()) {
 545                         continue;
 546                 }
 547
 548                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*j)->decoder);
 549                 ContentTime const from = dcp_to_content_subtitle (*j, time);
 550                 /* XXX: this video_frame_rate() should be the rate that the subtitle content has been prepared for */
 551                 ContentTime const to = from + ContentTime::from_frames (1, _film->video_frame_rate ());
 552
 553                 list<ContentImageSubtitle> image = subtitle_decoder->get_image_subtitles (ContentTimePeriod (from, to), starting);
 554                 for (list<ContentImageSubtitle>::iterator i = image.begin(); i != image.end(); ++i) {
 555
 556                         /* Apply content's subtitle offsets */
 557                         i->sub.rectangle.x += subtitle_content->subtitle_x_offset ();
 558                         i->sub.rectangle.y += subtitle_content->subtitle_y_offset ();
 559
 560                         /* Apply content's subtitle scale */
 561                         i->sub.rectangle.width *= subtitle_content->subtitle_scale ();
 562                         i->sub.rectangle.height *= subtitle_content->subtitle_scale ();
 563
 564                         /* Apply a corrective translation to keep the subtitle centred after that scale */
 565                         i->sub.rectangle.x -= i->sub.rectangle.width * (subtitle_content->subtitle_scale() - 1);
 566                         i->sub.rectangle.y -= i->sub.rectangle.height * (subtitle_content->subtitle_scale() - 1);
 567
 568                         ps.image.push_back (i->sub);
 569                 }
 570
 571                 list<ContentTextSubtitle> text = subtitle_decoder->get_text_subtitles (ContentTimePeriod (from, to), starting);
 572                 for (list<ContentTextSubtitle>::const_iterator i = text.begin(); i != text.end(); ++i) {
 573                         copy (i->subs.begin(), i->subs.end(), back_inserter (ps.text));
 574                 }
 575         }
 576
 577         return ps;
 578 }