src/lib/player.cc

   1 /*
   2     Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <stdint.h>
  21 #include <algorithm>
  22 #include "player.h"
  23 #include "film.h"
  24 #include "ffmpeg_decoder.h"
  25 #include "audio_buffers.h"
  26 #include "ffmpeg_content.h"
  27 #include "image_decoder.h"
  28 #include "image_content.h"
  29 #include "sndfile_decoder.h"
  30 #include "sndfile_content.h"
  31 #include "subtitle_content.h"
  32 #include "subrip_decoder.h"
  33 #include "subrip_content.h"
  34 #include "playlist.h"
  35 #include "job.h"
  36 #include "image.h"
  37 #include "image_proxy.h"
  38 #include "ratio.h"
  39 #include "log.h"
  40 #include "scaler.h"
  41 #include "render_subtitles.h"
  42 #include "config.h"
  43 #include "content_video.h"
  44 #include "player_video_frame.h"
  45
  46 using std::list;
  47 using std::cout;
  48 using std::min;
  49 using std::max;
  50 using std::min;
  51 using std::vector;
  52 using std::pair;
  53 using std::map;
  54 using std::make_pair;
  55 using boost::shared_ptr;
  56 using boost::weak_ptr;
  57 using boost::dynamic_pointer_cast;
  58 using boost::optional;
  59
  60 Player::Player (shared_ptr<const Film> f, shared_ptr<const Playlist> p)
  61         : _film (f)
  62         , _playlist (p)
  63         , _have_valid_pieces (false)
  64         , _approximate_size (false)
  65         , _burn_subtitles (false)
  66 {
  67         _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
  68         _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3));
  69         _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
  70         set_video_container_size (_film->frame_size ());
  71 }
  72
  73 void
  74 Player::setup_pieces ()
  75 {
  76         list<shared_ptr<Piece> > old_pieces = _pieces;
  77         _pieces.clear ();
  78
  79         ContentList content = _playlist->content ();
  80
  81         for (ContentList::iterator i = content.begin(); i != content.end(); ++i) {
  82
  83                 if (!(*i)->paths_valid ()) {
  84                         continue;
  85                 }
  86
  87                 shared_ptr<Decoder> decoder;
  88                 optional<FrameRateChange> frc;
  89
  90                 /* Work out a FrameRateChange for the best overlap video for this content, in case we need it below */
  91                 DCPTime best_overlap_t;
  92                 shared_ptr<VideoContent> best_overlap;
  93                 for (ContentList::iterator j = content.begin(); j != content.end(); ++j) {
  94                         shared_ptr<VideoContent> vc = dynamic_pointer_cast<VideoContent> (*j);
  95                         if (!vc) {
  96                                 continue;
  97                         }
  98
  99                         DCPTime const overlap = max (vc->position(), (*i)->position()) - min (vc->end(), (*i)->end());
 100                         if (overlap > best_overlap_t) {
 101                                 best_overlap = vc;
 102                                 best_overlap_t = overlap;
 103                         }
 104                 }
 105
 106                 optional<FrameRateChange> best_overlap_frc;
 107                 if (best_overlap) {
 108                         best_overlap_frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ());
 109                 } else {
 110                         /* No video overlap; e.g. if the DCP is just audio */
 111                         best_overlap_frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ());
 112                 }
 113
 114                 /* FFmpeg */
 115                 shared_ptr<const FFmpegContent> fc = dynamic_pointer_cast<const FFmpegContent> (*i);
 116                 if (fc) {
 117                         decoder.reset (new FFmpegDecoder (fc, _film->log()));
 118                         frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate());
 119                 }
 120
 121                 /* ImageContent */
 122                 shared_ptr<const ImageContent> ic = dynamic_pointer_cast<const ImageContent> (*i);
 123                 if (ic) {
 124                         /* See if we can re-use an old ImageDecoder */
 125                         for (list<shared_ptr<Piece> >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) {
 126                                 shared_ptr<ImageDecoder> imd = dynamic_pointer_cast<ImageDecoder> ((*j)->decoder);
 127                                 if (imd && imd->content() == ic) {
 128                                         decoder = imd;
 129                                 }
 130                         }
 131
 132                         if (!decoder) {
 133                                 decoder.reset (new ImageDecoder (ic));
 134                         }
 135
 136                         frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate());
 137                 }
 138
 139                 /* SndfileContent */
 140                 shared_ptr<const SndfileContent> sc = dynamic_pointer_cast<const SndfileContent> (*i);
 141                 if (sc) {
 142                         decoder.reset (new SndfileDecoder (sc));
 143                         frc = best_overlap_frc;
 144                 }
 145
 146                 /* SubRipContent */
 147                 shared_ptr<const SubRipContent> rc = dynamic_pointer_cast<const SubRipContent> (*i);
 148                 if (rc) {
 149                         decoder.reset (new SubRipDecoder (rc));
 150                         frc = best_overlap_frc;
 151                 }
 152
 153                 _pieces.push_back (shared_ptr<Piece> (new Piece (*i, decoder, frc.get ())));
 154         }
 155
 156         _have_valid_pieces = true;
 157 }
 158
 159 void
 160 Player::content_changed (weak_ptr<Content> w, int property, bool frequent)
 161 {
 162         shared_ptr<Content> c = w.lock ();
 163         if (!c) {
 164                 return;
 165         }
 166
 167         if (
 168                 property == ContentProperty::POSITION ||
 169                 property == ContentProperty::LENGTH ||
 170                 property == ContentProperty::TRIM_START ||
 171                 property == ContentProperty::TRIM_END ||
 172                 property == ContentProperty::PATH ||
 173                 property == VideoContentProperty::VIDEO_FRAME_TYPE
 174                 ) {
 175
 176                 _have_valid_pieces = false;
 177                 Changed (frequent);
 178
 179         } else if (
 180                 property == SubtitleContentProperty::SUBTITLE_X_OFFSET ||
 181                 property == SubtitleContentProperty::SUBTITLE_Y_OFFSET ||
 182                 property == SubtitleContentProperty::SUBTITLE_SCALE ||
 183                 property == VideoContentProperty::VIDEO_CROP ||
 184                 property == VideoContentProperty::VIDEO_SCALE ||
 185                 property == VideoContentProperty::VIDEO_FRAME_RATE
 186                 ) {
 187
 188                 Changed (frequent);
 189         }
 190 }
 191
 192 void
 193 Player::playlist_changed ()
 194 {
 195         _have_valid_pieces = false;
 196         Changed (false);
 197 }
 198
 199 void
 200 Player::set_video_container_size (dcp::Size s)
 201 {
 202         _video_container_size = s;
 203
 204         _black_image.reset (new Image (PIX_FMT_RGB24, _video_container_size, true));
 205         _black_image->make_black ();
 206 }
 207
 208 void
 209 Player::film_changed (Film::Property p)
 210 {
 211         /* Here we should notice Film properties that affect our output, and
 212            alert listeners that our output now would be different to how it was
 213            last time we were run.
 214         */
 215
 216         if (p == Film::SCALER || p == Film::WITH_SUBTITLES || p == Film::CONTAINER || p == Film::VIDEO_FRAME_RATE) {
 217                 Changed (false);
 218         }
 219 }
 220
 221 list<PositionImage>
 222 Player::process_content_image_subtitles (shared_ptr<SubtitleContent> content, list<shared_ptr<ContentImageSubtitle> > subs) const
 223 {
 224         list<PositionImage> all;
 225
 226         for (list<shared_ptr<ContentImageSubtitle> >::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 227                 if (!(*i)->image) {
 228                         continue;
 229                 }
 230
 231                 dcpomatic::Rect<double> in_rect = (*i)->rectangle;
 232                 dcp::Size scaled_size;
 233
 234                 in_rect.x += content->subtitle_x_offset ();
 235                 in_rect.y += content->subtitle_y_offset ();
 236
 237                 /* We will scale the subtitle up to fit _video_container_size, and also by the additional subtitle_scale */
 238                 scaled_size.width = in_rect.width * _video_container_size.width * content->subtitle_scale ();
 239                 scaled_size.height = in_rect.height * _video_container_size.height * content->subtitle_scale ();
 240
 241                 /* Then we need a corrective translation, consisting of two parts:
 242                  *
 243                  * 1.  that which is the result of the scaling of the subtitle by _video_container_size; this will be
 244                  *     rect.x * _video_container_size.width and rect.y * _video_container_size.height.
 245                  *
 246                  * 2.  that to shift the origin of the scale by subtitle_scale to the centre of the subtitle; this will be
 247                  *     (width_before_subtitle_scale * (1 - subtitle_scale) / 2) and
 248                  *     (height_before_subtitle_scale * (1 - subtitle_scale) / 2).
 249                  *
 250                  * Combining these two translations gives these expressions.
 251                  */
 252
 253                 all.push_back (
 254                         PositionImage (
 255                                 (*i)->image->scale (
 256                                         scaled_size,
 257                                         Scaler::from_id ("bicubic"),
 258                                         (*i)->image->pixel_format (),
 259                                         true
 260                                         ),
 261                                 Position<int> (
 262                                         rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - content->subtitle_scale ()) / 2))),
 263                                         rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - content->subtitle_scale ()) / 2)))
 264                                         )
 265                                 )
 266                         );
 267         }
 268
 269         return all;
 270 }
 271
 272 list<PositionImage>
 273 Player::process_content_text_subtitles (list<shared_ptr<ContentTextSubtitle> > sub) const
 274 {
 275         list<PositionImage> all;
 276         for (list<shared_ptr<ContentTextSubtitle> >::const_iterator i = sub.begin(); i != sub.end(); ++i) {
 277                 if (!(*i)->subs.empty ()) {
 278                         all.push_back (render_subtitles ((*i)->subs, _video_container_size));
 279                 }
 280         }
 281
 282         return all;
 283 }
 284
 285 void
 286 Player::set_approximate_size ()
 287 {
 288         _approximate_size = true;
 289 }
 290
 291 shared_ptr<PlayerVideoFrame>
 292 Player::black_player_video_frame () const
 293 {
 294         return shared_ptr<PlayerVideoFrame> (
 295                 new PlayerVideoFrame (
 296                         shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)),
 297                         Crop (),
 298                         _video_container_size,
 299                         _video_container_size,
 300                         Scaler::from_id ("bicubic"),
 301                         EYES_BOTH,
 302                         PART_WHOLE,
 303                         Config::instance()->colour_conversions().front().conversion
 304                 )
 305         );
 306 }
 307
 308 shared_ptr<PlayerVideoFrame>
 309 Player::content_to_player_video_frame (
 310         shared_ptr<VideoContent> content,
 311         ContentVideo content_video,
 312         list<shared_ptr<Piece> > subs,
 313         DCPTime time,
 314         dcp::Size image_size) const
 315 {
 316         shared_ptr<PlayerVideoFrame> pvf (
 317                 new PlayerVideoFrame (
 318                         content_video.image,
 319                         content->crop (),
 320                         image_size,
 321                         _video_container_size,
 322                         _film->scaler(),
 323                         content_video.eyes,
 324                         content_video.part,
 325                         content->colour_conversion ()
 326                         )
 327                 );
 328
 329
 330         /* Add subtitles */
 331
 332         list<PositionImage> sub_images;
 333
 334         for (list<shared_ptr<Piece> >::const_iterator i = subs.begin(); i != subs.end(); ++i) {
 335                 shared_ptr<SubtitleDecoder> subtitle_decoder = dynamic_pointer_cast<SubtitleDecoder> ((*i)->decoder);
 336                 shared_ptr<SubtitleContent> subtitle_content = dynamic_pointer_cast<SubtitleContent> ((*i)->content);
 337                 ContentTime const from = dcp_to_content_subtitle (*i, time);
 338                 ContentTime const to = from + ContentTime::from_frames (1, content->video_frame_rate ());
 339
 340                 list<shared_ptr<ContentImageSubtitle> > image_subtitles = subtitle_decoder->get_image_subtitles (from, to);
 341                 if (!image_subtitles.empty ()) {
 342                         list<PositionImage> im = process_content_image_subtitles (
 343                                 subtitle_content,
 344                                 image_subtitles
 345                                 );
 346
 347                         copy (im.begin(), im.end(), back_inserter (sub_images));
 348                 }
 349
 350                 if (_burn_subtitles) {
 351                         list<shared_ptr<ContentTextSubtitle> > text_subtitles = subtitle_decoder->get_text_subtitles (from, to);
 352                         if (!text_subtitles.empty ()) {
 353                                 list<PositionImage> im = process_content_text_subtitles (text_subtitles);
 354                                 copy (im.begin(), im.end(), back_inserter (sub_images));
 355                         }
 356                 }
 357         }
 358
 359         if (!sub_images.empty ()) {
 360                 pvf->set_subtitle (merge (sub_images));
 361         }
 362
 363         return pvf;
 364 }
 365
 366 /** @return All PlayerVideoFrames at the given time (there may be two frames for 3D) */
 367 list<shared_ptr<PlayerVideoFrame> >
 368 Player::get_video (DCPTime time, bool accurate)
 369 {
 370         if (!_have_valid_pieces) {
 371                 setup_pieces ();
 372         }
 373
 374         list<shared_ptr<Piece> > ov = overlaps<VideoContent> (
 375                 time,
 376                 time + DCPTime::from_frames (1, _film->video_frame_rate ())
 377                 );
 378
 379         list<shared_ptr<PlayerVideoFrame> > pvf;
 380
 381         if (ov.empty ()) {
 382                 /* No video content at this time */
 383                 pvf.push_back (black_player_video_frame ());
 384                 return pvf;
 385         }
 386
 387         /* Create a PlayerVideoFrame from the content's video at this time */
 388
 389         shared_ptr<Piece> piece = ov.back ();
 390         shared_ptr<VideoDecoder> decoder = dynamic_pointer_cast<VideoDecoder> (piece->decoder);
 391         assert (decoder);
 392         shared_ptr<VideoContent> content = dynamic_pointer_cast<VideoContent> (piece->content);
 393         assert (content);
 394
 395         list<ContentVideo> content_video = decoder->get_video (dcp_to_content_video (piece, time), accurate);
 396         if (content_video.empty ()) {
 397                 pvf.push_back (black_player_video_frame ());
 398                 return pvf;
 399         }
 400
 401         dcp::Size image_size = content->scale().size (content, _video_container_size, _film->frame_size ());
 402         if (_approximate_size) {
 403                 image_size.width &= ~3;
 404                 image_size.height &= ~3;
 405         }
 406
 407         for (list<ContentVideo>::const_iterator i = content_video.begin(); i != content_video.end(); ++i) {
 408                 list<shared_ptr<Piece> > subs = overlaps<SubtitleContent> (
 409                         time,
 410                         time + DCPTime::from_frames (1, _film->video_frame_rate ())
 411                         );
 412
 413                 pvf.push_back (content_to_player_video_frame (content, *i, subs, time, image_size));
 414         }
 415
 416         return pvf;
 417 }
 418
 419 shared_ptr<AudioBuffers>
 420 Player::get_audio (DCPTime time, DCPTime length, bool accurate)
 421 {
 422         if (!_have_valid_pieces) {
 423                 setup_pieces ();
 424         }
 425
 426         AudioFrame const length_frames = length.frames (_film->audio_frame_rate ());
 427
 428         shared_ptr<AudioBuffers> audio (new AudioBuffers (_film->audio_channels(), length_frames));
 429         audio->make_silent ();
 430
 431         list<shared_ptr<Piece> > ov = overlaps<AudioContent> (time, time + length);
 432         if (ov.empty ()) {
 433                 return audio;
 434         }
 435
 436         for (list<shared_ptr<Piece> >::iterator i = ov.begin(); i != ov.end(); ++i) {
 437
 438                 shared_ptr<AudioContent> content = dynamic_pointer_cast<AudioContent> ((*i)->content);
 439                 assert (content);
 440                 shared_ptr<AudioDecoder> decoder = dynamic_pointer_cast<AudioDecoder> ((*i)->decoder);
 441                 assert (decoder);
 442
 443                 if (content->audio_frame_rate() == 0) {
 444                         /* This AudioContent has no audio (e.g. if it is an FFmpegContent with no
 445                          * audio stream).
 446                          */
 447                         continue;
 448                 }
 449
 450                 /* The time that we should request from the content */
 451                 DCPTime request = time - DCPTime::from_seconds (content->audio_delay() / 1000.0);
 452                 DCPTime offset;
 453                 if (request < DCPTime ()) {
 454                         /* We went off the start of the content, so we will need to offset
 455                            the stuff we get back.
 456                         */
 457                         offset = -request;
 458                         request = DCPTime ();
 459                 }
 460
 461                 AudioFrame const content_frame = dcp_to_content_audio (*i, request);
 462
 463                 /* Audio from this piece's decoder (which might be more or less than what we asked for) */
 464                 shared_ptr<ContentAudio> all = decoder->get_audio (content_frame, length_frames, accurate);
 465
 466                 /* Gain */
 467                 if (content->audio_gain() != 0) {
 468                         shared_ptr<AudioBuffers> gain (new AudioBuffers (all->audio));
 469                         gain->apply_gain (content->audio_gain ());
 470                         all->audio = gain;
 471                 }
 472
 473                 /* Remap channels */
 474                 shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), all->audio->frames()));
 475                 dcp_mapped->make_silent ();
 476                 AudioMapping map = content->audio_mapping ();
 477                 for (int i = 0; i < map.content_channels(); ++i) {
 478                         for (int j = 0; j < _film->audio_channels(); ++j) {
 479                                 if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
 480                                         dcp_mapped->accumulate_channel (
 481                                                 all->audio.get(),
 482                                                 i,
 483                                                 j,
 484                                                 map.get (i, static_cast<dcp::Channel> (j))
 485                                                 );
 486                                 }
 487                         }
 488                 }
 489
 490                 all->audio = dcp_mapped;
 491
 492                 audio->accumulate_frames (
 493                         all->audio.get(),
 494                         content_frame - all->frame,
 495                         offset.frames (_film->audio_frame_rate()),
 496                         min (AudioFrame (all->audio->frames()), length_frames) - offset.frames (_film->audio_frame_rate ())
 497                         );
 498         }
 499
 500         return audio;
 501 }
 502
 503 VideoFrame
 504 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
 505 {
 506         /* s is the offset of t from the start position of this content */
 507         DCPTime s = t - piece->content->position ();
 508         s = DCPTime (max (int64_t (0), s.get ()));
 509         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 510
 511         /* Convert this to the content frame */
 512         return DCPTime (s + piece->content->trim_start()).frames (_film->video_frame_rate()) * piece->frc.factor ();
 513 }
 514
 515 AudioFrame
 516 Player::dcp_to_content_audio (shared_ptr<const Piece> piece, DCPTime t) const
 517 {
 518         /* s is the offset of t from the start position of this content */
 519         DCPTime s = t - piece->content->position ();
 520         s = DCPTime (max (int64_t (0), s.get ()));
 521         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 522
 523         /* Convert this to the content frame */
 524         return DCPTime (s + piece->content->trim_start()).frames (_film->audio_frame_rate());
 525 }
 526
 527 ContentTime
 528 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
 529 {
 530         /* s is the offset of t from the start position of this content */
 531         DCPTime s = t - piece->content->position ();
 532         s = DCPTime (max (int64_t (0), s.get ()));
 533         s = DCPTime (min (piece->content->length_after_trim().get(), s.get()));
 534
 535         return ContentTime (s, piece->frc);
 536 }
 537
 538 void
 539 PlayerStatistics::dump (shared_ptr<Log> log) const
 540 {
 541         log->log (String::compose ("Video: %1 good %2 skipped %3 black %4 repeat", video.good, video.skip, video.black, video.repeat));
 542         log->log (String::compose ("Audio: %1 good %2 skipped %3 silence", audio.good, audio.skip, audio.silence.seconds()));
 543 }
 544
 545 PlayerStatistics const &
 546 Player::statistics () const
 547 {
 548         return _statistics;
 549 }