2 Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 /** @file src/image.cc
23 * @brief A class to describe a video image.
27 #include "compose.hpp"
28 #include "dcpomatic_assert.h"
29 #include "dcpomatic_socket.h"
30 #include "enum_indexed_vector.h"
31 #include "exceptions.h"
33 #include "maths_util.h"
34 #include "memory_util.h"
37 #include <dcp/rgb_xyz.h>
38 #include <dcp/transfer_function.h>
39 #include <dcp/warnings.h>
40 LIBDCP_DISABLE_WARNINGS
42 #include <libavutil/frame.h>
43 #include <libavutil/pixdesc.h>
44 #include <libavutil/pixfmt.h>
45 #include <libswscale/swscale.h>
47 LIBDCP_ENABLE_WARNINGS
48 #if HAVE_VALGRIND_MEMCHECK_H
49 #include <valgrind/memcheck.h>
60 using std::make_shared;
63 using std::runtime_error;
64 using std::shared_ptr;
69 /** The memory alignment, in bytes, used for each row of an image if Alignment::PADDED is requested */
70 int constexpr ALIGNMENT = 64;
72 /* U/V black value for 8-bit colour */
73 static uint8_t const eight_bit_uv = (1 << 7) - 1;
74 /* U/V black value for 9-bit colour */
75 static uint16_t const nine_bit_uv = (1 << 8) - 1;
76 /* U/V black value for 10-bit colour */
77 static uint16_t const ten_bit_uv = (1 << 9) - 1;
78 /* U/V black value for 16-bit colour */
79 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
83 Image::vertical_factor (int n) const
89 auto d = av_pix_fmt_desc_get(_pixel_format);
91 throw PixelFormatError ("line_factor()", _pixel_format);
94 return lrintf(powf(2.0f, d->log2_chroma_h));
98 Image::horizontal_factor (int n) const
104 auto d = av_pix_fmt_desc_get(_pixel_format);
106 throw PixelFormatError ("sample_size()", _pixel_format);
109 return lrintf(powf(2.0f, d->log2_chroma_w));
113 /** @param n Component index.
114 * @return Number of samples (i.e. pixels, unless sub-sampled) in each direction for this component.
117 Image::sample_size (int n) const
120 lrint (ceil(static_cast<double>(size().width) / horizontal_factor(n))),
121 lrint (ceil(static_cast<double>(size().height) / vertical_factor(n)))
126 /** @return Number of planes */
128 Image::planes () const
130 if (_pixel_format == AV_PIX_FMT_PAL8) {
134 auto d = av_pix_fmt_desc_get(_pixel_format);
136 throw PixelFormatError ("planes()", _pixel_format);
139 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
143 return d->nb_components;
149 round_width_for_subsampling (int p, AVPixFmtDescriptor const * desc)
151 return p & ~ ((1 << desc->log2_chroma_w) - 1);
157 round_height_for_subsampling (int p, AVPixFmtDescriptor const * desc)
159 return p & ~ ((1 << desc->log2_chroma_h) - 1);
163 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size'.
164 * @param crop Amount to crop by.
165 * @param inter_size Size to scale the cropped image to.
166 * @param out_size Size of output frame; if this is larger than inter_size there will be black padding.
167 * @param yuv_to_rgb YUV to RGB transformation to use, if required.
168 * @param video_range Video range of the image.
169 * @param out_format Output pixel format.
170 * @param out_aligned true to make the output image aligned.
171 * @param out_video_range Video range to use for the output image.
172 * @param fast Try to be fast at the possible expense of quality; at present this means using
173 * fast bilinear rather than bicubic scaling.
176 Image::crop_scale_window (
178 dcp::Size inter_size,
180 dcp::YUVToRGB yuv_to_rgb,
181 VideoRange video_range,
182 AVPixelFormat out_format,
183 VideoRange out_video_range,
184 Alignment out_alignment,
188 /* Empirical testing suggests that sws_scale() will crash if
189 the input image is not padded.
191 DCPOMATIC_ASSERT (alignment() == Alignment::PADDED);
193 DCPOMATIC_ASSERT (out_size.width >= inter_size.width);
194 DCPOMATIC_ASSERT (out_size.height >= inter_size.height);
196 auto out = make_shared<Image>(out_format, out_size, out_alignment);
199 auto in_desc = av_pix_fmt_desc_get (_pixel_format);
201 throw PixelFormatError ("crop_scale_window()", _pixel_format);
204 /* Round down so that we crop only the number of pixels that is straightforward
205 * considering any subsampling.
208 round_width_for_subsampling(crop.left, in_desc),
209 round_width_for_subsampling(crop.right, in_desc),
210 round_height_for_subsampling(crop.top, in_desc),
211 round_height_for_subsampling(crop.bottom, in_desc)
214 /* Also check that we aren't cropping more image than there actually is */
215 if ((corrected_crop.left + corrected_crop.right) >= (size().width - 4)) {
216 corrected_crop.left = 0;
217 corrected_crop.right = size().width - 4;
220 if ((corrected_crop.top + corrected_crop.bottom) >= (size().height - 4)) {
221 corrected_crop.top = 0;
222 corrected_crop.bottom = size().height - 4;
225 /* Size of the image after any crop */
226 auto const cropped_size = corrected_crop.apply (size());
228 /* Scale context for a scale from cropped_size to inter_size */
229 auto scale_context = sws_getContext (
230 cropped_size.width, cropped_size.height, pixel_format(),
231 inter_size.width, inter_size.height, out_format,
232 fast ? SWS_FAST_BILINEAR : SWS_BICUBIC, 0, 0, 0
235 if (!scale_context) {
236 throw runtime_error (N_("Could not allocate SwsContext"));
239 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
240 EnumIndexedVector<int, dcp::YUVToRGB> lut;
241 lut[dcp::YUVToRGB::REC601] = SWS_CS_ITU601;
242 lut[dcp::YUVToRGB::REC709] = SWS_CS_ITU709;
243 lut[dcp::YUVToRGB::REC2020] = SWS_CS_BT2020;
245 /* The 3rd parameter here is:
246 0 -> source range MPEG (i.e. "video", 16-235)
247 1 -> source range JPEG (i.e. "full", 0-255)
249 0 -> destination range MPEG (i.e. "video", 16-235)
250 1 -> destination range JPEG (i.e. "full", 0-255)
252 But remember: sws_setColorspaceDetails ignores these
253 parameters unless the both source and destination images
254 are isYUV or isGray. (If either is not, it uses video range).
256 sws_setColorspaceDetails (
258 sws_getCoefficients(lut[yuv_to_rgb]), video_range == VideoRange::VIDEO ? 0 : 1,
259 sws_getCoefficients(lut[yuv_to_rgb]), out_video_range == VideoRange::VIDEO ? 0 : 1,
263 /* Prepare input data pointers with crop */
264 uint8_t* scale_in_data[planes()];
265 for (int c = 0; c < planes(); ++c) {
266 int const x = lrintf(bytes_per_pixel(c) * corrected_crop.left);
267 scale_in_data[c] = data()[c] + x + stride()[c] * (corrected_crop.top / vertical_factor(c));
270 auto out_desc = av_pix_fmt_desc_get (out_format);
272 throw PixelFormatError ("crop_scale_window()", out_format);
275 /* Corner of the image within out_size */
276 Position<int> const corner (
277 round_width_for_subsampling((out_size.width - inter_size.width) / 2, out_desc),
278 round_height_for_subsampling((out_size.height - inter_size.height) / 2, out_desc)
281 uint8_t* scale_out_data[out->planes()];
282 for (int c = 0; c < out->planes(); ++c) {
283 int const x = lrintf(out->bytes_per_pixel(c) * corner.x);
284 scale_out_data[c] = out->data()[c] + x + out->stride()[c] * (corner.y / out->vertical_factor(c));
289 scale_in_data, stride(),
290 0, cropped_size.height,
291 scale_out_data, out->stride()
294 sws_freeContext (scale_context);
296 /* There are some cases where there will be unwanted image data left in the image at this point:
298 * 1. When we are cropping without any scaling or pixel format conversion.
299 * 2. When we are scaling to certain sizes and placing the result into a larger
302 * Clear out the sides of the image to take care of those cases.
304 auto const pad = (out_size.width - inter_size.width) / 2;
305 out->make_part_black(0, pad);
306 out->make_part_black(corner.x + inter_size.width, pad);
309 video_range == VideoRange::VIDEO &&
310 out_video_range == VideoRange::FULL &&
311 av_pix_fmt_desc_get(_pixel_format)->flags & AV_PIX_FMT_FLAG_RGB
313 /* libswscale will not convert video range for RGB sources, so we have to do it ourselves */
314 out->video_range_to_full_range ();
322 Image::convert_pixel_format (dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, Alignment out_alignment, bool fast) const
324 return scale(size(), yuv_to_rgb, out_format, out_alignment, fast);
328 /** @param out_size Size to scale to.
329 * @param yuv_to_rgb YUVToRGB transform transform to use, if required.
330 * @param out_format Output pixel format.
331 * @param out_alignment Output alignment.
332 * @param fast Try to be fast at the possible expense of quality; at present this means using
333 * fast bilinear rather than bicubic scaling.
336 Image::scale (dcp::Size out_size, dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, Alignment out_alignment, bool fast) const
338 /* Empirical testing suggests that sws_scale() will crash if
339 the input image alignment is not PADDED.
341 DCPOMATIC_ASSERT (alignment() == Alignment::PADDED);
343 auto scaled = make_shared<Image>(out_format, out_size, out_alignment);
344 auto scale_context = sws_getContext (
345 size().width, size().height, pixel_format(),
346 out_size.width, out_size.height, out_format,
347 (fast ? SWS_FAST_BILINEAR : SWS_BICUBIC) | SWS_ACCURATE_RND, 0, 0, 0
350 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
351 EnumIndexedVector<int, dcp::YUVToRGB> lut;
352 lut[dcp::YUVToRGB::REC601] = SWS_CS_ITU601;
353 lut[dcp::YUVToRGB::REC709] = SWS_CS_ITU709;
354 lut[dcp::YUVToRGB::REC2020] = SWS_CS_BT2020;
356 /* The 3rd parameter here is:
357 0 -> source range MPEG (i.e. "video", 16-235)
358 1 -> source range JPEG (i.e. "full", 0-255)
360 0 -> destination range MPEG (i.e. "video", 16-235)
361 1 -> destination range JPEG (i.e. "full", 0-255)
363 But remember: sws_setColorspaceDetails ignores these
364 parameters unless the corresponding image isYUV or isGray.
365 (If it's neither, it uses video range).
367 sws_setColorspaceDetails (
369 sws_getCoefficients(lut[yuv_to_rgb]), 0,
370 sws_getCoefficients(lut[yuv_to_rgb]), 0,
378 scaled->data(), scaled->stride()
381 sws_freeContext (scale_context);
387 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
389 Image::yuv_16_black (uint16_t v, bool alpha)
391 memset (data()[0], 0, sample_size(0).height * stride()[0]);
392 for (int i = 1; i < 3; ++i) {
393 auto p = reinterpret_cast<int16_t*> (data()[i]);
394 int const lines = sample_size(i).height;
395 for (int y = 0; y < lines; ++y) {
396 /* We divide by 2 here because we are writing 2 bytes at a time */
397 for (int x = 0; x < line_size()[i] / 2; ++x) {
400 p += stride()[i] / 2;
405 memset (data()[3], 0, sample_size(3).height * stride()[3]);
411 Image::swap_16 (uint16_t v)
413 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
418 Image::make_part_black (int const start, int const width)
420 auto y_part = [&]() {
421 int const bpp = bytes_per_pixel(0);
422 int const h = sample_size(0).height;
423 int const s = stride()[0];
425 for (int y = 0; y < h; ++y) {
426 memset (p + start * bpp, 0, width * bpp);
431 switch (_pixel_format) {
432 case AV_PIX_FMT_RGB24:
433 case AV_PIX_FMT_ARGB:
434 case AV_PIX_FMT_RGBA:
435 case AV_PIX_FMT_ABGR:
436 case AV_PIX_FMT_BGRA:
437 case AV_PIX_FMT_RGB555LE:
438 case AV_PIX_FMT_RGB48LE:
439 case AV_PIX_FMT_RGB48BE:
440 case AV_PIX_FMT_XYZ12LE:
442 int const h = sample_size(0).height;
443 int const bpp = bytes_per_pixel(0);
444 int const s = stride()[0];
445 uint8_t* p = data()[0];
446 for (int y = 0; y < h; y++) {
447 memset (p + start * bpp, 0, width * bpp);
452 case AV_PIX_FMT_YUV420P:
455 for (int i = 1; i < 3; ++i) {
457 int const h = sample_size(i).height;
458 for (int y = 0; y < h; ++y) {
459 for (int x = start / 2; x < (start + width) / 2; ++x) {
467 case AV_PIX_FMT_YUV422P10LE:
470 for (int i = 1; i < 3; ++i) {
471 auto p = reinterpret_cast<int16_t*>(data()[i]);
472 int const h = sample_size(i).height;
473 for (int y = 0; y < h; ++y) {
474 for (int x = start / 2; x < (start + width) / 2; ++x) {
477 p += stride()[i] / 2;
482 case AV_PIX_FMT_YUV444P10LE:
485 for (int i = 1; i < 3; ++i) {
486 auto p = reinterpret_cast<int16_t*>(data()[i]);
487 int const h = sample_size(i).height;
488 for (int y = 0; y < h; ++y) {
489 for (int x = start; x < (start + width); ++x) {
492 p += stride()[i] / 2;
498 throw PixelFormatError ("make_part_black()", _pixel_format);
506 switch (_pixel_format) {
507 case AV_PIX_FMT_YUV420P:
508 case AV_PIX_FMT_YUV422P:
509 case AV_PIX_FMT_YUV444P:
510 case AV_PIX_FMT_YUV411P:
511 memset (data()[0], 0, sample_size(0).height * stride()[0]);
512 memset (data()[1], eight_bit_uv, sample_size(1).height * stride()[1]);
513 memset (data()[2], eight_bit_uv, sample_size(2).height * stride()[2]);
516 case AV_PIX_FMT_YUVJ420P:
517 case AV_PIX_FMT_YUVJ422P:
518 case AV_PIX_FMT_YUVJ444P:
519 memset (data()[0], 0, sample_size(0).height * stride()[0]);
520 memset (data()[1], eight_bit_uv + 1, sample_size(1).height * stride()[1]);
521 memset (data()[2], eight_bit_uv + 1, sample_size(2).height * stride()[2]);
524 case AV_PIX_FMT_YUV422P9LE:
525 case AV_PIX_FMT_YUV444P9LE:
526 yuv_16_black (nine_bit_uv, false);
529 case AV_PIX_FMT_YUV422P9BE:
530 case AV_PIX_FMT_YUV444P9BE:
531 yuv_16_black (swap_16 (nine_bit_uv), false);
534 case AV_PIX_FMT_YUV422P10LE:
535 case AV_PIX_FMT_YUV444P10LE:
536 yuv_16_black (ten_bit_uv, false);
539 case AV_PIX_FMT_YUV422P16LE:
540 case AV_PIX_FMT_YUV444P16LE:
541 yuv_16_black (sixteen_bit_uv, false);
544 case AV_PIX_FMT_YUV444P10BE:
545 case AV_PIX_FMT_YUV422P10BE:
546 yuv_16_black (swap_16 (ten_bit_uv), false);
549 case AV_PIX_FMT_YUVA420P9BE:
550 case AV_PIX_FMT_YUVA422P9BE:
551 case AV_PIX_FMT_YUVA444P9BE:
552 yuv_16_black (swap_16 (nine_bit_uv), true);
555 case AV_PIX_FMT_YUVA420P9LE:
556 case AV_PIX_FMT_YUVA422P9LE:
557 case AV_PIX_FMT_YUVA444P9LE:
558 yuv_16_black (nine_bit_uv, true);
561 case AV_PIX_FMT_YUVA420P10BE:
562 case AV_PIX_FMT_YUVA422P10BE:
563 case AV_PIX_FMT_YUVA444P10BE:
564 yuv_16_black (swap_16 (ten_bit_uv), true);
567 case AV_PIX_FMT_YUVA420P10LE:
568 case AV_PIX_FMT_YUVA422P10LE:
569 case AV_PIX_FMT_YUVA444P10LE:
570 yuv_16_black (ten_bit_uv, true);
573 case AV_PIX_FMT_YUVA420P16BE:
574 case AV_PIX_FMT_YUVA422P16BE:
575 case AV_PIX_FMT_YUVA444P16BE:
576 yuv_16_black (swap_16 (sixteen_bit_uv), true);
579 case AV_PIX_FMT_YUVA420P16LE:
580 case AV_PIX_FMT_YUVA422P16LE:
581 case AV_PIX_FMT_YUVA444P16LE:
582 yuv_16_black (sixteen_bit_uv, true);
585 case AV_PIX_FMT_RGB24:
586 case AV_PIX_FMT_ARGB:
587 case AV_PIX_FMT_RGBA:
588 case AV_PIX_FMT_ABGR:
589 case AV_PIX_FMT_BGRA:
590 case AV_PIX_FMT_RGB555LE:
591 case AV_PIX_FMT_RGB48LE:
592 case AV_PIX_FMT_RGB48BE:
593 case AV_PIX_FMT_XYZ12LE:
594 memset (data()[0], 0, sample_size(0).height * stride()[0]);
597 case AV_PIX_FMT_UYVY422:
599 int const Y = sample_size(0).height;
600 int const X = line_size()[0];
601 uint8_t* p = data()[0];
602 for (int y = 0; y < Y; ++y) {
603 for (int x = 0; x < X / 4; ++x) {
604 *p++ = eight_bit_uv; // Cb
606 *p++ = eight_bit_uv; // Cr
614 throw PixelFormatError ("make_black()", _pixel_format);
620 Image::make_transparent ()
622 if (_pixel_format != AV_PIX_FMT_BGRA && _pixel_format != AV_PIX_FMT_RGBA) {
623 throw PixelFormatError ("make_transparent()", _pixel_format);
626 memset (data()[0], 0, sample_size(0).height * stride()[0]);
631 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
633 /* We're blending RGBA or BGRA images */
634 DCPOMATIC_ASSERT (other->pixel_format() == AV_PIX_FMT_BGRA || other->pixel_format() == AV_PIX_FMT_RGBA);
635 int const blue = other->pixel_format() == AV_PIX_FMT_BGRA ? 0 : 2;
636 int const red = other->pixel_format() == AV_PIX_FMT_BGRA ? 2 : 0;
638 int const other_bpp = 4;
640 int start_tx = position.x;
644 start_ox = -start_tx;
648 int start_ty = position.y;
652 start_oy = -start_ty;
656 switch (_pixel_format) {
657 case AV_PIX_FMT_RGB24:
659 /* Going onto RGB24. First byte is red, second green, third blue */
660 int const this_bpp = 3;
661 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
662 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
663 uint8_t* op = other->data()[0] + oy * other->stride()[0];
664 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
665 float const alpha = float (op[3]) / 255;
666 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
667 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
668 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
676 case AV_PIX_FMT_BGRA:
678 int const this_bpp = 4;
679 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
680 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
681 uint8_t* op = other->data()[0] + oy * other->stride()[0];
682 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
683 float const alpha = float (op[3]) / 255;
684 tp[0] = op[blue] * alpha + tp[0] * (1 - alpha);
685 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
686 tp[2] = op[red] * alpha + tp[2] * (1 - alpha);
687 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
695 case AV_PIX_FMT_RGBA:
697 int const this_bpp = 4;
698 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
699 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
700 uint8_t* op = other->data()[0] + oy * other->stride()[0];
701 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
702 float const alpha = float (op[3]) / 255;
703 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
704 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
705 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
706 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
714 case AV_PIX_FMT_RGB48LE:
716 int const this_bpp = 6;
717 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
718 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
719 uint8_t* op = other->data()[0] + oy * other->stride()[0];
720 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
721 float const alpha = float (op[3]) / 255;
722 /* Blend high bytes */
723 tp[1] = op[red] * alpha + tp[1] * (1 - alpha);
724 tp[3] = op[1] * alpha + tp[3] * (1 - alpha);
725 tp[5] = op[blue] * alpha + tp[5] * (1 - alpha);
733 case AV_PIX_FMT_XYZ12LE:
735 auto conv = dcp::ColourConversion::srgb_to_xyz();
736 double fast_matrix[9];
737 dcp::combined_rgb_to_xyz (conv, fast_matrix);
738 auto lut_in = conv.in()->lut(0, 1, 8, false);
739 auto lut_out = conv.out()->lut(0, 1, 16, true);
740 int const this_bpp = 6;
741 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
742 uint16_t* tp = reinterpret_cast<uint16_t*> (data()[0] + ty * stride()[0] + start_tx * this_bpp);
743 uint8_t* op = other->data()[0] + oy * other->stride()[0];
744 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
745 float const alpha = float (op[3]) / 255;
747 /* Convert sRGB to XYZ; op is BGRA. First, input gamma LUT */
748 double const r = lut_in[op[red]];
749 double const g = lut_in[op[1]];
750 double const b = lut_in[op[blue]];
752 /* RGB to XYZ, including Bradford transform and DCI companding */
753 double const x = max(0.0, min(1.0, r * fast_matrix[0] + g * fast_matrix[1] + b * fast_matrix[2]));
754 double const y = max(0.0, min(1.0, r * fast_matrix[3] + g * fast_matrix[4] + b * fast_matrix[5]));
755 double const z = max(0.0, min(1.0, r * fast_matrix[6] + g * fast_matrix[7] + b * fast_matrix[8]));
757 /* Out gamma LUT and blend */
758 tp[0] = lrint(lut_out[lrint(x * 65535)] * 65535) * alpha + tp[0] * (1 - alpha);
759 tp[1] = lrint(lut_out[lrint(y * 65535)] * 65535) * alpha + tp[1] * (1 - alpha);
760 tp[2] = lrint(lut_out[lrint(z * 65535)] * 65535) * alpha + tp[2] * (1 - alpha);
768 case AV_PIX_FMT_YUV420P:
770 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
771 dcp::Size const ts = size();
772 dcp::Size const os = yuv->size();
773 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
774 int const hty = ty / 2;
775 int const hoy = oy / 2;
776 uint8_t* tY = data()[0] + (ty * stride()[0]) + start_tx;
777 uint8_t* tU = data()[1] + (hty * stride()[1]) + start_tx / 2;
778 uint8_t* tV = data()[2] + (hty * stride()[2]) + start_tx / 2;
779 uint8_t* oY = yuv->data()[0] + (oy * yuv->stride()[0]) + start_ox;
780 uint8_t* oU = yuv->data()[1] + (hoy * yuv->stride()[1]) + start_ox / 2;
781 uint8_t* oV = yuv->data()[2] + (hoy * yuv->stride()[2]) + start_ox / 2;
782 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
783 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
784 float const a = float(alpha[3]) / 255;
785 *tY = *oY * a + *tY * (1 - a);
786 *tU = *oU * a + *tU * (1 - a);
787 *tV = *oV * a + *tV * (1 - a);
803 case AV_PIX_FMT_YUV420P10:
805 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
806 dcp::Size const ts = size();
807 dcp::Size const os = yuv->size();
808 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
809 int const hty = ty / 2;
810 int const hoy = oy / 2;
811 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
812 uint16_t* tU = ((uint16_t *) (data()[1] + (hty * stride()[1]))) + start_tx / 2;
813 uint16_t* tV = ((uint16_t *) (data()[2] + (hty * stride()[2]))) + start_tx / 2;
814 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
815 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (hoy * yuv->stride()[1]))) + start_ox / 2;
816 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (hoy * yuv->stride()[2]))) + start_ox / 2;
817 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
818 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
819 float const a = float(alpha[3]) / 255;
820 *tY = *oY * a + *tY * (1 - a);
821 *tU = *oU * a + *tU * (1 - a);
822 *tV = *oV * a + *tV * (1 - a);
838 case AV_PIX_FMT_YUV422P10LE:
840 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
841 dcp::Size const ts = size();
842 dcp::Size const os = yuv->size();
843 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
844 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
845 uint16_t* tU = ((uint16_t *) (data()[1] + (ty * stride()[1]))) + start_tx / 2;
846 uint16_t* tV = ((uint16_t *) (data()[2] + (ty * stride()[2]))) + start_tx / 2;
847 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
848 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (oy * yuv->stride()[1]))) + start_ox / 2;
849 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (oy * yuv->stride()[2]))) + start_ox / 2;
850 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
851 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
852 float const a = float(alpha[3]) / 255;
853 *tY = *oY * a + *tY * (1 - a);
854 *tU = *oU * a + *tU * (1 - a);
855 *tV = *oV * a + *tV * (1 - a);
872 throw PixelFormatError ("alpha_blend()", _pixel_format);
878 Image::copy (shared_ptr<const Image> other, Position<int> position)
880 /* Only implemented for RGB24 onto RGB24 so far */
881 DCPOMATIC_ASSERT (_pixel_format == AV_PIX_FMT_RGB24 && other->pixel_format() == AV_PIX_FMT_RGB24);
882 DCPOMATIC_ASSERT (position.x >= 0 && position.y >= 0);
884 int const N = min (position.x + other->size().width, size().width) - position.x;
885 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
886 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
887 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
888 memcpy (tp, op, N * 3);
894 Image::read_from_socket (shared_ptr<Socket> socket)
896 for (int i = 0; i < planes(); ++i) {
897 uint8_t* p = data()[i];
898 int const lines = sample_size(i).height;
899 for (int y = 0; y < lines; ++y) {
900 socket->read (p, line_size()[i]);
908 Image::write_to_socket (shared_ptr<Socket> socket) const
910 for (int i = 0; i < planes(); ++i) {
911 uint8_t* p = data()[i];
912 int const lines = sample_size(i).height;
913 for (int y = 0; y < lines; ++y) {
914 socket->write (p, line_size()[i]);
922 Image::bytes_per_pixel (int c) const
924 auto d = av_pix_fmt_desc_get(_pixel_format);
926 throw PixelFormatError ("bytes_per_pixel()", _pixel_format);
933 float bpp[4] = { 0, 0, 0, 0 };
935 #ifdef DCPOMATIC_HAVE_AVCOMPONENTDESCRIPTOR_DEPTH_MINUS1
936 bpp[0] = floor ((d->comp[0].depth_minus1 + 8) / 8);
937 if (d->nb_components > 1) {
938 bpp[1] = floor ((d->comp[1].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
940 if (d->nb_components > 2) {
941 bpp[2] = floor ((d->comp[2].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
943 if (d->nb_components > 3) {
944 bpp[3] = floor ((d->comp[3].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
947 bpp[0] = floor ((d->comp[0].depth + 7) / 8);
948 if (d->nb_components > 1) {
949 bpp[1] = floor ((d->comp[1].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
951 if (d->nb_components > 2) {
952 bpp[2] = floor ((d->comp[2].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
954 if (d->nb_components > 3) {
955 bpp[3] = floor ((d->comp[3].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
959 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
960 /* Not planar; sum them up */
961 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
968 /** Construct a Image of a given size and format, allocating memory
971 * @param p Pixel format.
972 * @param s Size in pixels.
973 * @param alignment PADDED to make each row of this image aligned to a ALIGNMENT-byte boundary, otherwise COMPACT.
975 Image::Image (AVPixelFormat p, dcp::Size s, Alignment alignment)
978 , _alignment (alignment)
987 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
988 _data[0] = _data[1] = _data[2] = _data[3] = 0;
990 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
991 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
993 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
994 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
996 auto stride_round_up = [](int stride, int t) {
997 int const a = stride + (t - 1);
1001 for (int i = 0; i < planes(); ++i) {
1002 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
1003 _stride[i] = stride_round_up (_line_size[i], _alignment == Alignment::PADDED ? ALIGNMENT : 1);
1005 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
1006 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
1007 Hence on the last pixel of the last line it reads over the end of
1008 the actual data by 1 byte. If the width of an image is a multiple
1009 of the stride alignment there will be no padding at the end of image lines.
1010 OS X crashes on this illegal read, though other operating systems don't
1011 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
1012 for that instruction to read safely.
1014 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
1015 over-reads by more then _avx. I can't follow the code to work out how much,
1016 so I'll just over-allocate by ALIGNMENT bytes and have done with it. Empirical
1017 testing suggests that it works.
1019 In addition to these concerns, we may read/write as much as a whole extra line
1020 at the end of each plane in cases where we are messing with offsets in order to
1021 do pad or crop. To solve this we over-allocate by an extra _stride[i] bytes.
1023 As an example: we may write to images starting at an offset so we get some padding.
1024 Hence we want to write in the following pattern:
1026 block start write start line end
1027 |..(padding)..|<------line-size------------->|..(padding)..|
1028 |..(padding)..|<------line-size------------->|..(padding)..|
1029 |..(padding)..|<------line-size------------->|..(padding)..|
1031 where line-size is of the smaller (inter_size) image and the full padded line length is that of
1032 out_size. To get things to work we have to tell FFmpeg that the stride is that of out_size.
1033 However some parts of FFmpeg (notably rgb48Toxyz12 in swscale.c) process data for the full
1034 specified *stride*. This does not matter until we get to the last line:
1036 block start write start line end
1037 |..(padding)..|<------line-size------------->|XXXwrittenXXX|
1038 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXX|
1039 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXXXXXwrittenXXX
1042 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
1043 #if HAVE_VALGRIND_MEMCHECK_H
1044 /* The data between the end of the line size and the stride is undefined but processed by
1045 libswscale, causing lots of valgrind errors. Mark it all defined to quell these errors.
1047 VALGRIND_MAKE_MEM_DEFINED (_data[i], _stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
1053 Image::Image (Image const & other)
1054 : std::enable_shared_from_this<Image>(other)
1055 , _size (other._size)
1056 , _pixel_format (other._pixel_format)
1057 , _alignment (other._alignment)
1061 for (int i = 0; i < planes(); ++i) {
1062 uint8_t* p = _data[i];
1063 uint8_t* q = other._data[i];
1064 int const lines = sample_size(i).height;
1065 for (int j = 0; j < lines; ++j) {
1066 memcpy (p, q, _line_size[i]);
1068 q += other.stride()[i];
1074 Image::Image (AVFrame const * frame, Alignment alignment)
1075 : _size (frame->width, frame->height)
1076 , _pixel_format (static_cast<AVPixelFormat>(frame->format))
1077 , _alignment (alignment)
1079 DCPOMATIC_ASSERT (_pixel_format != AV_PIX_FMT_NONE);
1083 for (int i = 0; i < planes(); ++i) {
1084 uint8_t* p = _data[i];
1085 uint8_t* q = frame->data[i];
1086 int const lines = sample_size(i).height;
1087 for (int j = 0; j < lines; ++j) {
1088 memcpy (p, q, _line_size[i]);
1090 /* AVFrame's linesize is what we call `stride' */
1091 q += frame->linesize[i];
1097 Image::Image (shared_ptr<const Image> other, Alignment alignment)
1098 : _size (other->_size)
1099 , _pixel_format (other->_pixel_format)
1100 , _alignment (alignment)
1104 for (int i = 0; i < planes(); ++i) {
1105 DCPOMATIC_ASSERT (line_size()[i] == other->line_size()[i]);
1106 uint8_t* p = _data[i];
1107 uint8_t* q = other->data()[i];
1108 int const lines = sample_size(i).height;
1109 for (int j = 0; j < lines; ++j) {
1110 memcpy (p, q, line_size()[i]);
1112 q += other->stride()[i];
1119 Image::operator= (Image const & other)
1121 if (this == &other) {
1132 Image::swap (Image & other)
1134 std::swap (_size, other._size);
1135 std::swap (_pixel_format, other._pixel_format);
1137 for (int i = 0; i < 4; ++i) {
1138 std::swap (_data[i], other._data[i]);
1139 std::swap (_line_size[i], other._line_size[i]);
1140 std::swap (_stride[i], other._stride[i]);
1143 std::swap (_alignment, other._alignment);
1149 for (int i = 0; i < planes(); ++i) {
1154 av_free (_line_size);
1160 Image::data () const
1167 Image::line_size () const
1174 Image::stride () const
1181 Image::size () const
1188 Image::alignment () const
1195 merge (list<PositionImage> images, Image::Alignment alignment)
1197 if (images.empty ()) {
1201 if (images.size() == 1) {
1202 images.front().image = Image::ensure_alignment(images.front().image, alignment);
1203 return images.front();
1206 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
1207 for (auto const& i: images) {
1208 all.extend (dcpomatic::Rect<int>(i.position, i.image->size().width, i.image->size().height));
1211 auto merged = make_shared<Image>(images.front().image->pixel_format(), dcp::Size(all.width, all.height), alignment);
1212 merged->make_transparent ();
1213 for (auto const& i: images) {
1214 merged->alpha_blend (i.image, i.position - all.position());
1217 return PositionImage (merged, all.position ());
1222 operator== (Image const & a, Image const & b)
1224 if (a.planes() != b.planes() || a.pixel_format() != b.pixel_format() || a.alignment() != b.alignment()) {
1228 for (int c = 0; c < a.planes(); ++c) {
1229 if (a.sample_size(c).height != b.sample_size(c).height || a.line_size()[c] != b.line_size()[c] || a.stride()[c] != b.stride()[c]) {
1233 uint8_t* p = a.data()[c];
1234 uint8_t* q = b.data()[c];
1235 int const lines = a.sample_size(c).height;
1236 for (int y = 0; y < lines; ++y) {
1237 if (memcmp (p, q, a.line_size()[c]) != 0) {
1251 * @param f Amount to fade by; 0 is black, 1 is no fade.
1254 Image::fade (float f)
1256 /* U/V black value for 8-bit colour */
1257 static int const eight_bit_uv = (1 << 7) - 1;
1258 /* U/V black value for 10-bit colour */
1259 static uint16_t const ten_bit_uv = (1 << 9) - 1;
1261 switch (_pixel_format) {
1262 case AV_PIX_FMT_YUV420P:
1265 uint8_t* p = data()[0];
1266 int const lines = sample_size(0).height;
1267 for (int y = 0; y < lines; ++y) {
1269 for (int x = 0; x < line_size()[0]; ++x) {
1270 *q = int(float(*q) * f);
1277 for (int c = 1; c < 3; ++c) {
1278 uint8_t* p = data()[c];
1279 int const lines = sample_size(c).height;
1280 for (int y = 0; y < lines; ++y) {
1282 for (int x = 0; x < line_size()[c]; ++x) {
1283 *q = eight_bit_uv + int((int(*q) - eight_bit_uv) * f);
1293 case AV_PIX_FMT_RGB24:
1296 uint8_t* p = data()[0];
1297 int const lines = sample_size(0).height;
1298 for (int y = 0; y < lines; ++y) {
1300 for (int x = 0; x < line_size()[0]; ++x) {
1301 *q = int (float (*q) * f);
1309 case AV_PIX_FMT_XYZ12LE:
1310 case AV_PIX_FMT_RGB48LE:
1311 /* 16-bit little-endian */
1312 for (int c = 0; c < 3; ++c) {
1313 int const stride_pixels = stride()[c] / 2;
1314 int const line_size_pixels = line_size()[c] / 2;
1315 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1316 int const lines = sample_size(c).height;
1317 for (int y = 0; y < lines; ++y) {
1319 for (int x = 0; x < line_size_pixels; ++x) {
1320 *q = int (float (*q) * f);
1328 case AV_PIX_FMT_YUV422P10LE:
1332 int const stride_pixels = stride()[0] / 2;
1333 int const line_size_pixels = line_size()[0] / 2;
1334 uint16_t* p = reinterpret_cast<uint16_t*> (data()[0]);
1335 int const lines = sample_size(0).height;
1336 for (int y = 0; y < lines; ++y) {
1338 for (int x = 0; x < line_size_pixels; ++x) {
1339 *q = int(float(*q) * f);
1347 for (int c = 1; c < 3; ++c) {
1348 int const stride_pixels = stride()[c] / 2;
1349 int const line_size_pixels = line_size()[c] / 2;
1350 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1351 int const lines = sample_size(c).height;
1352 for (int y = 0; y < lines; ++y) {
1354 for (int x = 0; x < line_size_pixels; ++x) {
1355 *q = ten_bit_uv + int((int(*q) - ten_bit_uv) * f);
1366 throw PixelFormatError ("fade()", _pixel_format);
1371 shared_ptr<const Image>
1372 Image::ensure_alignment (shared_ptr<const Image> image, Image::Alignment alignment)
1374 if (image->alignment() == alignment) {
1378 return make_shared<Image>(image, alignment);
1383 Image::memory_used () const
1386 for (int i = 0; i < planes(); ++i) {
1387 m += _stride[i] * sample_size(i).height;
1394 Image::video_range_to_full_range ()
1396 switch (_pixel_format) {
1397 case AV_PIX_FMT_RGB24:
1399 float const factor = 256.0 / 219.0;
1400 uint8_t* p = data()[0];
1401 int const lines = sample_size(0).height;
1402 for (int y = 0; y < lines; ++y) {
1404 for (int x = 0; x < line_size()[0]; ++x) {
1405 *q = clamp(lrintf((*q - 16) * factor), 0L, 255L);
1412 case AV_PIX_FMT_RGB48LE:
1414 float const factor = 65536.0 / 56064.0;
1415 uint16_t* p = reinterpret_cast<uint16_t*>(data()[0]);
1416 int const lines = sample_size(0).height;
1417 for (int y = 0; y < lines; ++y) {
1419 int const line_size_pixels = line_size()[0] / 2;
1420 for (int x = 0; x < line_size_pixels; ++x) {
1421 *q = clamp(lrintf((*q - 4096) * factor), 0L, 65535L);
1424 p += stride()[0] / 2;
1428 case AV_PIX_FMT_GBRP12LE:
1430 float const factor = 4096.0 / 3504.0;
1431 for (int c = 0; c < 3; ++c) {
1432 uint16_t* p = reinterpret_cast<uint16_t*>(data()[c]);
1433 int const lines = sample_size(c).height;
1434 for (int y = 0; y < lines; ++y) {
1436 int const line_size_pixels = line_size()[c] / 2;
1437 for (int x = 0; x < line_size_pixels; ++x) {
1438 *q = clamp(lrintf((*q - 256) * factor), 0L, 4095L);
1446 throw PixelFormatError ("video_range_to_full_range()", _pixel_format);