2 Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 /** @file src/image.cc
23 * @brief A class to describe a video image.
27 #include "compose.hpp"
28 #include "dcpomatic_assert.h"
29 #include "dcpomatic_socket.h"
30 #include "exceptions.h"
32 #include "maths_util.h"
33 #include "memory_util.h"
36 #include <dcp/rgb_xyz.h>
37 #include <dcp/transfer_function.h>
38 #include <dcp/warnings.h>
39 LIBDCP_DISABLE_WARNINGS
41 #include <libavutil/frame.h>
42 #include <libavutil/pixdesc.h>
43 #include <libavutil/pixfmt.h>
44 #include <libswscale/swscale.h>
46 LIBDCP_ENABLE_WARNINGS
47 #if HAVE_VALGRIND_MEMCHECK_H
48 #include <valgrind/memcheck.h>
59 using std::make_shared;
62 using std::runtime_error;
63 using std::shared_ptr;
68 /** The memory alignment, in bytes, used for each row of an image if Alignment::PADDED is requested */
69 int constexpr ALIGNMENT = 64;
71 /* U/V black value for 8-bit colour */
72 static uint8_t const eight_bit_uv = (1 << 7) - 1;
73 /* U/V black value for 9-bit colour */
74 static uint16_t const nine_bit_uv = (1 << 8) - 1;
75 /* U/V black value for 10-bit colour */
76 static uint16_t const ten_bit_uv = (1 << 9) - 1;
77 /* U/V black value for 16-bit colour */
78 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
82 Image::vertical_factor (int n) const
88 auto d = av_pix_fmt_desc_get(_pixel_format);
90 throw PixelFormatError ("line_factor()", _pixel_format);
93 return lrintf(powf(2.0f, d->log2_chroma_h));
97 Image::horizontal_factor (int n) const
103 auto d = av_pix_fmt_desc_get(_pixel_format);
105 throw PixelFormatError ("sample_size()", _pixel_format);
108 return lrintf(powf(2.0f, d->log2_chroma_w));
112 /** @param n Component index.
113 * @return Number of samples (i.e. pixels, unless sub-sampled) in each direction for this component.
116 Image::sample_size (int n) const
119 lrint (ceil(static_cast<double>(size().width) / horizontal_factor(n))),
120 lrint (ceil(static_cast<double>(size().height) / vertical_factor(n)))
125 /** @return Number of planes */
127 Image::planes () const
129 if (_pixel_format == AV_PIX_FMT_PAL8) {
133 auto d = av_pix_fmt_desc_get(_pixel_format);
135 throw PixelFormatError ("planes()", _pixel_format);
138 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
142 return d->nb_components;
148 round_width_for_subsampling (int p, AVPixFmtDescriptor const * desc)
150 return p & ~ ((1 << desc->log2_chroma_w) - 1);
156 round_height_for_subsampling (int p, AVPixFmtDescriptor const * desc)
158 return p & ~ ((1 << desc->log2_chroma_h) - 1);
162 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size'.
163 * @param crop Amount to crop by.
164 * @param inter_size Size to scale the cropped image to.
165 * @param out_size Size of output frame; if this is larger than inter_size there will be black padding.
166 * @param yuv_to_rgb YUV to RGB transformation to use, if required.
167 * @param video_range Video range of the image.
168 * @param out_format Output pixel format.
169 * @param out_aligned true to make the output image aligned.
170 * @param out_video_range Video range to use for the output image.
171 * @param fast Try to be fast at the possible expense of quality; at present this means using
172 * fast bilinear rather than bicubic scaling.
175 Image::crop_scale_window (
177 dcp::Size inter_size,
179 dcp::YUVToRGB yuv_to_rgb,
180 VideoRange video_range,
181 AVPixelFormat out_format,
182 VideoRange out_video_range,
183 Alignment out_alignment,
187 /* Empirical testing suggests that sws_scale() will crash if
188 the input image is not padded.
190 DCPOMATIC_ASSERT (alignment() == Alignment::PADDED);
192 DCPOMATIC_ASSERT (out_size.width >= inter_size.width);
193 DCPOMATIC_ASSERT (out_size.height >= inter_size.height);
195 auto out = make_shared<Image>(out_format, out_size, out_alignment);
198 auto in_desc = av_pix_fmt_desc_get (_pixel_format);
200 throw PixelFormatError ("crop_scale_window()", _pixel_format);
203 /* Round down so that we crop only the number of pixels that is straightforward
204 * considering any subsampling.
207 round_width_for_subsampling(crop.left, in_desc),
208 round_width_for_subsampling(crop.right, in_desc),
209 round_height_for_subsampling(crop.top, in_desc),
210 round_height_for_subsampling(crop.bottom, in_desc)
213 /* Also check that we aren't cropping more image than there actually is */
214 if ((corrected_crop.left + corrected_crop.right) >= (size().width - 4)) {
215 corrected_crop.left = 0;
216 corrected_crop.right = size().width - 4;
219 if ((corrected_crop.top + corrected_crop.bottom) >= (size().height - 4)) {
220 corrected_crop.top = 0;
221 corrected_crop.bottom = size().height - 4;
224 /* Size of the image after any crop */
225 auto const cropped_size = corrected_crop.apply (size());
227 /* Scale context for a scale from cropped_size to inter_size */
228 auto scale_context = sws_getContext (
229 cropped_size.width, cropped_size.height, pixel_format(),
230 inter_size.width, inter_size.height, out_format,
231 fast ? SWS_FAST_BILINEAR : SWS_BICUBIC, 0, 0, 0
234 if (!scale_context) {
235 throw runtime_error (N_("Could not allocate SwsContext"));
238 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
239 int const lut[static_cast<int>(dcp::YUVToRGB::COUNT)] = {
244 /* The 3rd parameter here is:
245 0 -> source range MPEG (i.e. "video", 16-235)
246 1 -> source range JPEG (i.e. "full", 0-255)
248 0 -> destination range MPEG (i.e. "video", 16-235)
249 1 -> destination range JPEG (i.e. "full", 0-255)
251 But remember: sws_setColorspaceDetails ignores these
252 parameters unless the both source and destination images
253 are isYUV or isGray. (If either is not, it uses video range).
255 sws_setColorspaceDetails (
257 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), video_range == VideoRange::VIDEO ? 0 : 1,
258 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), out_video_range == VideoRange::VIDEO ? 0 : 1,
262 /* Prepare input data pointers with crop */
263 uint8_t* scale_in_data[planes()];
264 for (int c = 0; c < planes(); ++c) {
265 int const x = lrintf(bytes_per_pixel(c) * corrected_crop.left);
266 scale_in_data[c] = data()[c] + x + stride()[c] * (corrected_crop.top / vertical_factor(c));
269 auto out_desc = av_pix_fmt_desc_get (out_format);
271 throw PixelFormatError ("crop_scale_window()", out_format);
274 /* Corner of the image within out_size */
275 Position<int> const corner (
276 round_width_for_subsampling((out_size.width - inter_size.width) / 2, out_desc),
277 round_height_for_subsampling((out_size.height - inter_size.height) / 2, out_desc)
280 uint8_t* scale_out_data[out->planes()];
281 for (int c = 0; c < out->planes(); ++c) {
282 int const x = lrintf(out->bytes_per_pixel(c) * corner.x);
283 scale_out_data[c] = out->data()[c] + x + out->stride()[c] * (corner.y / out->vertical_factor(c));
288 scale_in_data, stride(),
289 0, cropped_size.height,
290 scale_out_data, out->stride()
293 sws_freeContext (scale_context);
295 if (corrected_crop != Crop() && cropped_size == inter_size) {
296 /* We are cropping without any scaling or pixel format conversion, so FFmpeg may have left some
297 data behind in our image. Clear it out. It may get to the point where we should just stop
298 trying to be clever with cropping.
300 out->make_part_black (corner.x + cropped_size.width, out_size.width - cropped_size.width);
304 video_range == VideoRange::VIDEO &&
305 out_video_range == VideoRange::FULL &&
306 av_pix_fmt_desc_get(_pixel_format)->flags & AV_PIX_FMT_FLAG_RGB
308 /* libswscale will not convert video range for RGB sources, so we have to do it ourselves */
309 out->video_range_to_full_range ();
317 Image::convert_pixel_format (dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, Alignment out_alignment, bool fast) const
319 return scale(size(), yuv_to_rgb, out_format, out_alignment, fast);
323 /** @param out_size Size to scale to.
324 * @param yuv_to_rgb YUVToRGB transform transform to use, if required.
325 * @param out_format Output pixel format.
326 * @param out_aligment Output alignment.
327 * @param fast Try to be fast at the possible expense of quality; at present this means using
328 * fast bilinear rather than bicubic scaling.
331 Image::scale (dcp::Size out_size, dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, Alignment out_alignment, bool fast) const
333 /* Empirical testing suggests that sws_scale() will crash if
334 the input image alignment is not PADDED.
336 DCPOMATIC_ASSERT (alignment() == Alignment::PADDED);
338 auto scaled = make_shared<Image>(out_format, out_size, out_alignment);
339 auto scale_context = sws_getContext (
340 size().width, size().height, pixel_format(),
341 out_size.width, out_size.height, out_format,
342 (fast ? SWS_FAST_BILINEAR : SWS_BICUBIC) | SWS_ACCURATE_RND, 0, 0, 0
345 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
346 int const lut[static_cast<int>(dcp::YUVToRGB::COUNT)] = {
351 /* The 3rd parameter here is:
352 0 -> source range MPEG (i.e. "video", 16-235)
353 1 -> source range JPEG (i.e. "full", 0-255)
355 0 -> destination range MPEG (i.e. "video", 16-235)
356 1 -> destination range JPEG (i.e. "full", 0-255)
358 But remember: sws_setColorspaceDetails ignores these
359 parameters unless the corresponding image isYUV or isGray.
360 (If it's neither, it uses video range).
362 sws_setColorspaceDetails (
364 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), 0,
365 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), 0,
373 scaled->data(), scaled->stride()
376 sws_freeContext (scale_context);
382 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
384 Image::yuv_16_black (uint16_t v, bool alpha)
386 memset (data()[0], 0, sample_size(0).height * stride()[0]);
387 for (int i = 1; i < 3; ++i) {
388 auto p = reinterpret_cast<int16_t*> (data()[i]);
389 int const lines = sample_size(i).height;
390 for (int y = 0; y < lines; ++y) {
391 /* We divide by 2 here because we are writing 2 bytes at a time */
392 for (int x = 0; x < line_size()[i] / 2; ++x) {
395 p += stride()[i] / 2;
400 memset (data()[3], 0, sample_size(3).height * stride()[3]);
406 Image::swap_16 (uint16_t v)
408 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
413 Image::make_part_black (int const start, int const width)
415 auto y_part = [&]() {
416 int const bpp = bytes_per_pixel(0);
417 int const h = sample_size(0).height;
418 int const s = stride()[0];
420 for (int y = 0; y < h; ++y) {
421 memset (p + start * bpp, 0, width * bpp);
426 switch (_pixel_format) {
427 case AV_PIX_FMT_RGB24:
428 case AV_PIX_FMT_ARGB:
429 case AV_PIX_FMT_RGBA:
430 case AV_PIX_FMT_ABGR:
431 case AV_PIX_FMT_BGRA:
432 case AV_PIX_FMT_RGB555LE:
433 case AV_PIX_FMT_RGB48LE:
434 case AV_PIX_FMT_RGB48BE:
435 case AV_PIX_FMT_XYZ12LE:
437 int const h = sample_size(0).height;
438 int const bpp = bytes_per_pixel(0);
439 int const s = stride()[0];
440 uint8_t* p = data()[0];
441 for (int y = 0; y < h; y++) {
442 memset (p + start * bpp, 0, width * bpp);
447 case AV_PIX_FMT_YUV420P:
450 for (int i = 1; i < 3; ++i) {
452 int const h = sample_size(i).height;
453 for (int y = 0; y < h; ++y) {
454 for (int x = start / 2; x < (start + width) / 2; ++x) {
462 case AV_PIX_FMT_YUV422P10LE:
465 for (int i = 1; i < 3; ++i) {
466 auto p = reinterpret_cast<int16_t*>(data()[i]);
467 int const h = sample_size(i).height;
468 for (int y = 0; y < h; ++y) {
469 for (int x = start / 2; x < (start + width) / 2; ++x) {
472 p += stride()[i] / 2;
478 throw PixelFormatError ("make_part_black()", _pixel_format);
486 switch (_pixel_format) {
487 case AV_PIX_FMT_YUV420P:
488 case AV_PIX_FMT_YUV422P:
489 case AV_PIX_FMT_YUV444P:
490 case AV_PIX_FMT_YUV411P:
491 memset (data()[0], 0, sample_size(0).height * stride()[0]);
492 memset (data()[1], eight_bit_uv, sample_size(1).height * stride()[1]);
493 memset (data()[2], eight_bit_uv, sample_size(2).height * stride()[2]);
496 case AV_PIX_FMT_YUVJ420P:
497 case AV_PIX_FMT_YUVJ422P:
498 case AV_PIX_FMT_YUVJ444P:
499 memset (data()[0], 0, sample_size(0).height * stride()[0]);
500 memset (data()[1], eight_bit_uv + 1, sample_size(1).height * stride()[1]);
501 memset (data()[2], eight_bit_uv + 1, sample_size(2).height * stride()[2]);
504 case AV_PIX_FMT_YUV422P9LE:
505 case AV_PIX_FMT_YUV444P9LE:
506 yuv_16_black (nine_bit_uv, false);
509 case AV_PIX_FMT_YUV422P9BE:
510 case AV_PIX_FMT_YUV444P9BE:
511 yuv_16_black (swap_16 (nine_bit_uv), false);
514 case AV_PIX_FMT_YUV422P10LE:
515 case AV_PIX_FMT_YUV444P10LE:
516 yuv_16_black (ten_bit_uv, false);
519 case AV_PIX_FMT_YUV422P16LE:
520 case AV_PIX_FMT_YUV444P16LE:
521 yuv_16_black (sixteen_bit_uv, false);
524 case AV_PIX_FMT_YUV444P10BE:
525 case AV_PIX_FMT_YUV422P10BE:
526 yuv_16_black (swap_16 (ten_bit_uv), false);
529 case AV_PIX_FMT_YUVA420P9BE:
530 case AV_PIX_FMT_YUVA422P9BE:
531 case AV_PIX_FMT_YUVA444P9BE:
532 yuv_16_black (swap_16 (nine_bit_uv), true);
535 case AV_PIX_FMT_YUVA420P9LE:
536 case AV_PIX_FMT_YUVA422P9LE:
537 case AV_PIX_FMT_YUVA444P9LE:
538 yuv_16_black (nine_bit_uv, true);
541 case AV_PIX_FMT_YUVA420P10BE:
542 case AV_PIX_FMT_YUVA422P10BE:
543 case AV_PIX_FMT_YUVA444P10BE:
544 yuv_16_black (swap_16 (ten_bit_uv), true);
547 case AV_PIX_FMT_YUVA420P10LE:
548 case AV_PIX_FMT_YUVA422P10LE:
549 case AV_PIX_FMT_YUVA444P10LE:
550 yuv_16_black (ten_bit_uv, true);
553 case AV_PIX_FMT_YUVA420P16BE:
554 case AV_PIX_FMT_YUVA422P16BE:
555 case AV_PIX_FMT_YUVA444P16BE:
556 yuv_16_black (swap_16 (sixteen_bit_uv), true);
559 case AV_PIX_FMT_YUVA420P16LE:
560 case AV_PIX_FMT_YUVA422P16LE:
561 case AV_PIX_FMT_YUVA444P16LE:
562 yuv_16_black (sixteen_bit_uv, true);
565 case AV_PIX_FMT_RGB24:
566 case AV_PIX_FMT_ARGB:
567 case AV_PIX_FMT_RGBA:
568 case AV_PIX_FMT_ABGR:
569 case AV_PIX_FMT_BGRA:
570 case AV_PIX_FMT_RGB555LE:
571 case AV_PIX_FMT_RGB48LE:
572 case AV_PIX_FMT_RGB48BE:
573 case AV_PIX_FMT_XYZ12LE:
574 memset (data()[0], 0, sample_size(0).height * stride()[0]);
577 case AV_PIX_FMT_UYVY422:
579 int const Y = sample_size(0).height;
580 int const X = line_size()[0];
581 uint8_t* p = data()[0];
582 for (int y = 0; y < Y; ++y) {
583 for (int x = 0; x < X / 4; ++x) {
584 *p++ = eight_bit_uv; // Cb
586 *p++ = eight_bit_uv; // Cr
594 throw PixelFormatError ("make_black()", _pixel_format);
600 Image::make_transparent ()
602 if (_pixel_format != AV_PIX_FMT_BGRA && _pixel_format != AV_PIX_FMT_RGBA) {
603 throw PixelFormatError ("make_transparent()", _pixel_format);
606 memset (data()[0], 0, sample_size(0).height * stride()[0]);
611 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
613 /* We're blending RGBA or BGRA images */
614 DCPOMATIC_ASSERT (other->pixel_format() == AV_PIX_FMT_BGRA || other->pixel_format() == AV_PIX_FMT_RGBA);
615 int const blue = other->pixel_format() == AV_PIX_FMT_BGRA ? 0 : 2;
616 int const red = other->pixel_format() == AV_PIX_FMT_BGRA ? 2 : 0;
618 int const other_bpp = 4;
620 int start_tx = position.x;
624 start_ox = -start_tx;
628 int start_ty = position.y;
632 start_oy = -start_ty;
636 switch (_pixel_format) {
637 case AV_PIX_FMT_RGB24:
639 /* Going onto RGB24. First byte is red, second green, third blue */
640 int const this_bpp = 3;
641 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
642 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
643 uint8_t* op = other->data()[0] + oy * other->stride()[0];
644 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
645 float const alpha = float (op[3]) / 255;
646 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
647 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
648 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
656 case AV_PIX_FMT_BGRA:
658 int const this_bpp = 4;
659 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
660 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
661 uint8_t* op = other->data()[0] + oy * other->stride()[0];
662 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
663 float const alpha = float (op[3]) / 255;
664 tp[0] = op[blue] * alpha + tp[0] * (1 - alpha);
665 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
666 tp[2] = op[red] * alpha + tp[2] * (1 - alpha);
667 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
675 case AV_PIX_FMT_RGBA:
677 int const this_bpp = 4;
678 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
679 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
680 uint8_t* op = other->data()[0] + oy * other->stride()[0];
681 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
682 float const alpha = float (op[3]) / 255;
683 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
684 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
685 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
686 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
694 case AV_PIX_FMT_RGB48LE:
696 int const this_bpp = 6;
697 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
698 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
699 uint8_t* op = other->data()[0] + oy * other->stride()[0];
700 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
701 float const alpha = float (op[3]) / 255;
702 /* Blend high bytes */
703 tp[1] = op[red] * alpha + tp[1] * (1 - alpha);
704 tp[3] = op[1] * alpha + tp[3] * (1 - alpha);
705 tp[5] = op[blue] * alpha + tp[5] * (1 - alpha);
713 case AV_PIX_FMT_XYZ12LE:
715 auto conv = dcp::ColourConversion::srgb_to_xyz();
716 double fast_matrix[9];
717 dcp::combined_rgb_to_xyz (conv, fast_matrix);
718 double const * lut_in = conv.in()->lut (8, false);
719 double const * lut_out = conv.out()->lut (16, true);
720 int const this_bpp = 6;
721 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
722 uint16_t* tp = reinterpret_cast<uint16_t*> (data()[0] + ty * stride()[0] + start_tx * this_bpp);
723 uint8_t* op = other->data()[0] + oy * other->stride()[0];
724 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
725 float const alpha = float (op[3]) / 255;
727 /* Convert sRGB to XYZ; op is BGRA. First, input gamma LUT */
728 double const r = lut_in[op[red]];
729 double const g = lut_in[op[1]];
730 double const b = lut_in[op[blue]];
732 /* RGB to XYZ, including Bradford transform and DCI companding */
733 double const x = max (0.0, min (65535.0, r * fast_matrix[0] + g * fast_matrix[1] + b * fast_matrix[2]));
734 double const y = max (0.0, min (65535.0, r * fast_matrix[3] + g * fast_matrix[4] + b * fast_matrix[5]));
735 double const z = max (0.0, min (65535.0, r * fast_matrix[6] + g * fast_matrix[7] + b * fast_matrix[8]));
737 /* Out gamma LUT and blend */
738 tp[0] = lrint(lut_out[lrint(x)] * 65535) * alpha + tp[0] * (1 - alpha);
739 tp[1] = lrint(lut_out[lrint(y)] * 65535) * alpha + tp[1] * (1 - alpha);
740 tp[2] = lrint(lut_out[lrint(z)] * 65535) * alpha + tp[2] * (1 - alpha);
748 case AV_PIX_FMT_YUV420P:
750 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
751 dcp::Size const ts = size();
752 dcp::Size const os = yuv->size();
753 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
754 int const hty = ty / 2;
755 int const hoy = oy / 2;
756 uint8_t* tY = data()[0] + (ty * stride()[0]) + start_tx;
757 uint8_t* tU = data()[1] + (hty * stride()[1]) + start_tx / 2;
758 uint8_t* tV = data()[2] + (hty * stride()[2]) + start_tx / 2;
759 uint8_t* oY = yuv->data()[0] + (oy * yuv->stride()[0]) + start_ox;
760 uint8_t* oU = yuv->data()[1] + (hoy * yuv->stride()[1]) + start_ox / 2;
761 uint8_t* oV = yuv->data()[2] + (hoy * yuv->stride()[2]) + start_ox / 2;
762 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
763 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
764 float const a = float(alpha[3]) / 255;
765 *tY = *oY * a + *tY * (1 - a);
766 *tU = *oU * a + *tU * (1 - a);
767 *tV = *oV * a + *tV * (1 - a);
783 case AV_PIX_FMT_YUV420P10:
785 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
786 dcp::Size const ts = size();
787 dcp::Size const os = yuv->size();
788 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
789 int const hty = ty / 2;
790 int const hoy = oy / 2;
791 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
792 uint16_t* tU = ((uint16_t *) (data()[1] + (hty * stride()[1]))) + start_tx / 2;
793 uint16_t* tV = ((uint16_t *) (data()[2] + (hty * stride()[2]))) + start_tx / 2;
794 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
795 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (hoy * yuv->stride()[1]))) + start_ox / 2;
796 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (hoy * yuv->stride()[2]))) + start_ox / 2;
797 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
798 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
799 float const a = float(alpha[3]) / 255;
800 *tY = *oY * a + *tY * (1 - a);
801 *tU = *oU * a + *tU * (1 - a);
802 *tV = *oV * a + *tV * (1 - a);
818 case AV_PIX_FMT_YUV422P10LE:
820 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
821 dcp::Size const ts = size();
822 dcp::Size const os = yuv->size();
823 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
824 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
825 uint16_t* tU = ((uint16_t *) (data()[1] + (ty * stride()[1]))) + start_tx / 2;
826 uint16_t* tV = ((uint16_t *) (data()[2] + (ty * stride()[2]))) + start_tx / 2;
827 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
828 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (oy * yuv->stride()[1]))) + start_ox / 2;
829 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (oy * yuv->stride()[2]))) + start_ox / 2;
830 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
831 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
832 float const a = float(alpha[3]) / 255;
833 *tY = *oY * a + *tY * (1 - a);
834 *tU = *oU * a + *tU * (1 - a);
835 *tV = *oV * a + *tV * (1 - a);
852 throw PixelFormatError ("alpha_blend()", _pixel_format);
858 Image::copy (shared_ptr<const Image> other, Position<int> position)
860 /* Only implemented for RGB24 onto RGB24 so far */
861 DCPOMATIC_ASSERT (_pixel_format == AV_PIX_FMT_RGB24 && other->pixel_format() == AV_PIX_FMT_RGB24);
862 DCPOMATIC_ASSERT (position.x >= 0 && position.y >= 0);
864 int const N = min (position.x + other->size().width, size().width) - position.x;
865 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
866 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
867 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
868 memcpy (tp, op, N * 3);
874 Image::read_from_socket (shared_ptr<Socket> socket)
876 for (int i = 0; i < planes(); ++i) {
877 uint8_t* p = data()[i];
878 int const lines = sample_size(i).height;
879 for (int y = 0; y < lines; ++y) {
880 socket->read (p, line_size()[i]);
888 Image::write_to_socket (shared_ptr<Socket> socket) const
890 for (int i = 0; i < planes(); ++i) {
891 uint8_t* p = data()[i];
892 int const lines = sample_size(i).height;
893 for (int y = 0; y < lines; ++y) {
894 socket->write (p, line_size()[i]);
902 Image::bytes_per_pixel (int c) const
904 auto d = av_pix_fmt_desc_get(_pixel_format);
906 throw PixelFormatError ("bytes_per_pixel()", _pixel_format);
913 float bpp[4] = { 0, 0, 0, 0 };
915 #ifdef DCPOMATIC_HAVE_AVCOMPONENTDESCRIPTOR_DEPTH_MINUS1
916 bpp[0] = floor ((d->comp[0].depth_minus1 + 8) / 8);
917 if (d->nb_components > 1) {
918 bpp[1] = floor ((d->comp[1].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
920 if (d->nb_components > 2) {
921 bpp[2] = floor ((d->comp[2].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
923 if (d->nb_components > 3) {
924 bpp[3] = floor ((d->comp[3].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
927 bpp[0] = floor ((d->comp[0].depth + 7) / 8);
928 if (d->nb_components > 1) {
929 bpp[1] = floor ((d->comp[1].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
931 if (d->nb_components > 2) {
932 bpp[2] = floor ((d->comp[2].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
934 if (d->nb_components > 3) {
935 bpp[3] = floor ((d->comp[3].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
939 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
940 /* Not planar; sum them up */
941 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
948 /** Construct a Image of a given size and format, allocating memory
951 * @param p Pixel format.
952 * @param s Size in pixels.
953 * @param alignment PADDED to make each row of this image aligned to a ALIGNMENT-byte boundary, otherwise COMPACT.
955 Image::Image (AVPixelFormat p, dcp::Size s, Alignment alignment)
958 , _alignment (alignment)
967 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
968 _data[0] = _data[1] = _data[2] = _data[3] = 0;
970 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
971 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
973 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
974 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
976 auto stride_round_up = [](int stride, int t) {
977 int const a = stride + (t - 1);
981 for (int i = 0; i < planes(); ++i) {
982 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
983 _stride[i] = stride_round_up (_line_size[i], _alignment == Alignment::PADDED ? ALIGNMENT : 1);
985 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
986 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
987 Hence on the last pixel of the last line it reads over the end of
988 the actual data by 1 byte. If the width of an image is a multiple
989 of the stride alignment there will be no padding at the end of image lines.
990 OS X crashes on this illegal read, though other operating systems don't
991 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
992 for that instruction to read safely.
994 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
995 over-reads by more then _avx. I can't follow the code to work out how much,
996 so I'll just over-allocate by ALIGNMENT bytes and have done with it. Empirical
997 testing suggests that it works.
999 In addition to these concerns, we may read/write as much as a whole extra line
1000 at the end of each plane in cases where we are messing with offsets in order to
1001 do pad or crop. To solve this we over-allocate by an extra _stride[i] bytes.
1003 As an example: we may write to images starting at an offset so we get some padding.
1004 Hence we want to write in the following pattern:
1006 block start write start line end
1007 |..(padding)..|<------line-size------------->|..(padding)..|
1008 |..(padding)..|<------line-size------------->|..(padding)..|
1009 |..(padding)..|<------line-size------------->|..(padding)..|
1011 where line-size is of the smaller (inter_size) image and the full padded line length is that of
1012 out_size. To get things to work we have to tell FFmpeg that the stride is that of out_size.
1013 However some parts of FFmpeg (notably rgb48Toxyz12 in swscale.c) process data for the full
1014 specified *stride*. This does not matter until we get to the last line:
1016 block start write start line end
1017 |..(padding)..|<------line-size------------->|XXXwrittenXXX|
1018 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXX|
1019 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXXXXXwrittenXXX
1022 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
1023 #if HAVE_VALGRIND_MEMCHECK_H
1024 /* The data between the end of the line size and the stride is undefined but processed by
1025 libswscale, causing lots of valgrind errors. Mark it all defined to quell these errors.
1027 VALGRIND_MAKE_MEM_DEFINED (_data[i], _stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
1033 Image::Image (Image const & other)
1034 : std::enable_shared_from_this<Image>(other)
1035 , _size (other._size)
1036 , _pixel_format (other._pixel_format)
1037 , _alignment (other._alignment)
1041 for (int i = 0; i < planes(); ++i) {
1042 uint8_t* p = _data[i];
1043 uint8_t* q = other._data[i];
1044 int const lines = sample_size(i).height;
1045 for (int j = 0; j < lines; ++j) {
1046 memcpy (p, q, _line_size[i]);
1048 q += other.stride()[i];
1054 Image::Image (AVFrame const * frame, Alignment alignment)
1055 : _size (frame->width, frame->height)
1056 , _pixel_format (static_cast<AVPixelFormat>(frame->format))
1057 , _alignment (alignment)
1059 DCPOMATIC_ASSERT (_pixel_format != AV_PIX_FMT_NONE);
1063 for (int i = 0; i < planes(); ++i) {
1064 uint8_t* p = _data[i];
1065 uint8_t* q = frame->data[i];
1066 int const lines = sample_size(i).height;
1067 for (int j = 0; j < lines; ++j) {
1068 memcpy (p, q, _line_size[i]);
1070 /* AVFrame's linesize is what we call `stride' */
1071 q += frame->linesize[i];
1077 Image::Image (shared_ptr<const Image> other, Alignment alignment)
1078 : _size (other->_size)
1079 , _pixel_format (other->_pixel_format)
1080 , _alignment (alignment)
1084 for (int i = 0; i < planes(); ++i) {
1085 DCPOMATIC_ASSERT (line_size()[i] == other->line_size()[i]);
1086 uint8_t* p = _data[i];
1087 uint8_t* q = other->data()[i];
1088 int const lines = sample_size(i).height;
1089 for (int j = 0; j < lines; ++j) {
1090 memcpy (p, q, line_size()[i]);
1092 q += other->stride()[i];
1099 Image::operator= (Image const & other)
1101 if (this == &other) {
1112 Image::swap (Image & other)
1114 std::swap (_size, other._size);
1115 std::swap (_pixel_format, other._pixel_format);
1117 for (int i = 0; i < 4; ++i) {
1118 std::swap (_data[i], other._data[i]);
1119 std::swap (_line_size[i], other._line_size[i]);
1120 std::swap (_stride[i], other._stride[i]);
1123 std::swap (_alignment, other._alignment);
1129 for (int i = 0; i < planes(); ++i) {
1134 av_free (_line_size);
1140 Image::data () const
1147 Image::line_size () const
1154 Image::stride () const
1161 Image::size () const
1168 Image::alignment () const
1175 merge (list<PositionImage> images, Image::Alignment alignment)
1177 if (images.empty ()) {
1181 if (images.size() == 1) {
1182 images.front().image = Image::ensure_alignment(images.front().image, alignment);
1183 return images.front();
1186 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
1187 for (auto const& i: images) {
1188 all.extend (dcpomatic::Rect<int>(i.position, i.image->size().width, i.image->size().height));
1191 auto merged = make_shared<Image>(images.front().image->pixel_format(), dcp::Size(all.width, all.height), alignment);
1192 merged->make_transparent ();
1193 for (auto const& i: images) {
1194 merged->alpha_blend (i.image, i.position - all.position());
1197 return PositionImage (merged, all.position ());
1202 operator== (Image const & a, Image const & b)
1204 if (a.planes() != b.planes() || a.pixel_format() != b.pixel_format() || a.alignment() != b.alignment()) {
1208 for (int c = 0; c < a.planes(); ++c) {
1209 if (a.sample_size(c).height != b.sample_size(c).height || a.line_size()[c] != b.line_size()[c] || a.stride()[c] != b.stride()[c]) {
1213 uint8_t* p = a.data()[c];
1214 uint8_t* q = b.data()[c];
1215 int const lines = a.sample_size(c).height;
1216 for (int y = 0; y < lines; ++y) {
1217 if (memcmp (p, q, a.line_size()[c]) != 0) {
1231 * @param f Amount to fade by; 0 is black, 1 is no fade.
1234 Image::fade (float f)
1236 /* U/V black value for 8-bit colour */
1237 static int const eight_bit_uv = (1 << 7) - 1;
1238 /* U/V black value for 10-bit colour */
1239 static uint16_t const ten_bit_uv = (1 << 9) - 1;
1241 switch (_pixel_format) {
1242 case AV_PIX_FMT_YUV420P:
1245 uint8_t* p = data()[0];
1246 int const lines = sample_size(0).height;
1247 for (int y = 0; y < lines; ++y) {
1249 for (int x = 0; x < line_size()[0]; ++x) {
1250 *q = int(float(*q) * f);
1257 for (int c = 1; c < 3; ++c) {
1258 uint8_t* p = data()[c];
1259 int const lines = sample_size(c).height;
1260 for (int y = 0; y < lines; ++y) {
1262 for (int x = 0; x < line_size()[c]; ++x) {
1263 *q = eight_bit_uv + int((int(*q) - eight_bit_uv) * f);
1273 case AV_PIX_FMT_RGB24:
1276 uint8_t* p = data()[0];
1277 int const lines = sample_size(0).height;
1278 for (int y = 0; y < lines; ++y) {
1280 for (int x = 0; x < line_size()[0]; ++x) {
1281 *q = int (float (*q) * f);
1289 case AV_PIX_FMT_XYZ12LE:
1290 case AV_PIX_FMT_RGB48LE:
1291 /* 16-bit little-endian */
1292 for (int c = 0; c < 3; ++c) {
1293 int const stride_pixels = stride()[c] / 2;
1294 int const line_size_pixels = line_size()[c] / 2;
1295 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1296 int const lines = sample_size(c).height;
1297 for (int y = 0; y < lines; ++y) {
1299 for (int x = 0; x < line_size_pixels; ++x) {
1300 *q = int (float (*q) * f);
1308 case AV_PIX_FMT_YUV422P10LE:
1312 int const stride_pixels = stride()[0] / 2;
1313 int const line_size_pixels = line_size()[0] / 2;
1314 uint16_t* p = reinterpret_cast<uint16_t*> (data()[0]);
1315 int const lines = sample_size(0).height;
1316 for (int y = 0; y < lines; ++y) {
1318 for (int x = 0; x < line_size_pixels; ++x) {
1319 *q = int(float(*q) * f);
1327 for (int c = 1; c < 3; ++c) {
1328 int const stride_pixels = stride()[c] / 2;
1329 int const line_size_pixels = line_size()[c] / 2;
1330 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1331 int const lines = sample_size(c).height;
1332 for (int y = 0; y < lines; ++y) {
1334 for (int x = 0; x < line_size_pixels; ++x) {
1335 *q = ten_bit_uv + int((int(*q) - ten_bit_uv) * f);
1346 throw PixelFormatError ("fade()", _pixel_format);
1351 shared_ptr<const Image>
1352 Image::ensure_alignment (shared_ptr<const Image> image, Image::Alignment alignment)
1354 if (image->alignment() == alignment) {
1358 return make_shared<Image>(image, alignment);
1363 Image::memory_used () const
1366 for (int i = 0; i < planes(); ++i) {
1367 m += _stride[i] * sample_size(i).height;
1374 Image::video_range_to_full_range ()
1376 switch (_pixel_format) {
1377 case AV_PIX_FMT_RGB24:
1379 float const factor = 256.0 / 219.0;
1380 uint8_t* p = data()[0];
1381 int const lines = sample_size(0).height;
1382 for (int y = 0; y < lines; ++y) {
1384 for (int x = 0; x < line_size()[0]; ++x) {
1385 *q = clamp(lrintf((*q - 16) * factor), 0L, 255L);
1392 case AV_PIX_FMT_RGB48LE:
1394 float const factor = 65536.0 / 56064.0;
1395 uint16_t* p = reinterpret_cast<uint16_t*>(data()[0]);
1396 int const lines = sample_size(0).height;
1397 for (int y = 0; y < lines; ++y) {
1399 int const line_size_pixels = line_size()[0] / 2;
1400 for (int x = 0; x < line_size_pixels; ++x) {
1401 *q = clamp(lrintf((*q - 4096) * factor), 0L, 65535L);
1404 p += stride()[0] / 2;
1408 case AV_PIX_FMT_GBRP12LE:
1410 float const factor = 4096.0 / 3504.0;
1411 for (int c = 0; c < 3; ++c) {
1412 uint16_t* p = reinterpret_cast<uint16_t*>(data()[c]);
1413 int const lines = sample_size(c).height;
1414 for (int y = 0; y < lines; ++y) {
1416 int const line_size_pixels = line_size()[c] / 2;
1417 for (int x = 0; x < line_size_pixels; ++x) {
1418 *q = clamp(lrintf((*q - 256) * factor), 0L, 4095L);
1426 throw PixelFormatError ("video_range_to_full_range()", _pixel_format);