2 Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 /** @file src/image.cc
23 * @brief A class to describe a video image.
27 #include "compose.hpp"
28 #include "dcpomatic_assert.h"
29 #include "dcpomatic_socket.h"
30 #include "exceptions.h"
32 #include "maths_util.h"
37 #include <dcp/rgb_xyz.h>
38 #include <dcp/transfer_function.h>
39 DCPOMATIC_DISABLE_WARNINGS
41 #include <libavutil/frame.h>
42 #include <libavutil/pixdesc.h>
43 #include <libavutil/pixfmt.h>
44 #include <libswscale/swscale.h>
46 DCPOMATIC_ENABLE_WARNINGS
48 #if HAVE_VALGRIND_MEMCHECK_H
49 #include <valgrind/memcheck.h>
60 using std::make_shared;
63 using std::runtime_error;
64 using std::shared_ptr;
69 /** The memory alignment, in bytes, used for each row of an image if Alignment::PADDED is requested */
70 int constexpr ALIGNMENT = 64;
72 /* U/V black value for 8-bit colour */
73 static uint8_t const eight_bit_uv = (1 << 7) - 1;
74 /* U/V black value for 9-bit colour */
75 static uint16_t const nine_bit_uv = (1 << 8) - 1;
76 /* U/V black value for 10-bit colour */
77 static uint16_t const ten_bit_uv = (1 << 9) - 1;
78 /* U/V black value for 16-bit colour */
79 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
83 Image::vertical_factor (int n) const
89 auto d = av_pix_fmt_desc_get(_pixel_format);
91 throw PixelFormatError ("line_factor()", _pixel_format);
94 return lrintf(powf(2.0f, d->log2_chroma_h));
98 Image::horizontal_factor (int n) const
104 auto d = av_pix_fmt_desc_get(_pixel_format);
106 throw PixelFormatError ("sample_size()", _pixel_format);
109 return lrintf(powf(2.0f, d->log2_chroma_w));
113 /** @param n Component index.
114 * @return Number of samples (i.e. pixels, unless sub-sampled) in each direction for this component.
117 Image::sample_size (int n) const
120 lrint (ceil(static_cast<double>(size().width) / horizontal_factor(n))),
121 lrint (ceil(static_cast<double>(size().height) / vertical_factor(n)))
126 /** @return Number of planes */
128 Image::planes () const
130 if (_pixel_format == AV_PIX_FMT_PAL8) {
134 auto d = av_pix_fmt_desc_get(_pixel_format);
136 throw PixelFormatError ("planes()", _pixel_format);
139 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
143 return d->nb_components;
149 round_width_for_subsampling (int p, AVPixFmtDescriptor const * desc)
151 return p & ~ ((1 << desc->log2_chroma_w) - 1);
157 round_height_for_subsampling (int p, AVPixFmtDescriptor const * desc)
159 return p & ~ ((1 << desc->log2_chroma_h) - 1);
163 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size'.
164 * @param crop Amount to crop by.
165 * @param inter_size Size to scale the cropped image to.
166 * @param out_size Size of output frame; if this is larger than inter_size there will be black padding.
167 * @param yuv_to_rgb YUV to RGB transformation to use, if required.
168 * @param video_range Video range of the image.
169 * @param out_format Output pixel format.
170 * @param out_aligned true to make the output image aligned.
171 * @param out_video_range Video range to use for the output image.
172 * @param fast Try to be fast at the possible expense of quality; at present this means using
173 * fast bilinear rather than bicubic scaling.
176 Image::crop_scale_window (
178 dcp::Size inter_size,
180 dcp::YUVToRGB yuv_to_rgb,
181 VideoRange video_range,
182 AVPixelFormat out_format,
183 VideoRange out_video_range,
184 Alignment out_alignment,
188 /* Empirical testing suggests that sws_scale() will crash if
189 the input image is not padded.
191 DCPOMATIC_ASSERT (alignment() == Alignment::PADDED);
193 DCPOMATIC_ASSERT (out_size.width >= inter_size.width);
194 DCPOMATIC_ASSERT (out_size.height >= inter_size.height);
196 auto out = make_shared<Image>(out_format, out_size, out_alignment);
199 auto in_desc = av_pix_fmt_desc_get (_pixel_format);
201 throw PixelFormatError ("crop_scale_window()", _pixel_format);
204 /* Round down so that we crop only the number of pixels that is straightforward
205 * considering any subsampling.
208 round_width_for_subsampling(crop.left, in_desc),
209 round_width_for_subsampling(crop.right, in_desc),
210 round_height_for_subsampling(crop.top, in_desc),
211 round_height_for_subsampling(crop.bottom, in_desc)
214 /* Also check that we aren't cropping more image than there actually is */
215 if ((corrected_crop.left + corrected_crop.right) >= (size().width - 4)) {
216 corrected_crop.left = 0;
217 corrected_crop.right = size().width - 4;
220 if ((corrected_crop.top + corrected_crop.bottom) >= (size().height - 4)) {
221 corrected_crop.top = 0;
222 corrected_crop.bottom = size().height - 4;
225 /* Size of the image after any crop */
226 auto const cropped_size = corrected_crop.apply (size());
228 /* Scale context for a scale from cropped_size to inter_size */
229 auto scale_context = sws_getContext (
230 cropped_size.width, cropped_size.height, pixel_format(),
231 inter_size.width, inter_size.height, out_format,
232 fast ? SWS_FAST_BILINEAR : SWS_BICUBIC, 0, 0, 0
235 if (!scale_context) {
236 throw runtime_error (N_("Could not allocate SwsContext"));
239 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
240 int const lut[static_cast<int>(dcp::YUVToRGB::COUNT)] = {
245 /* The 3rd parameter here is:
246 0 -> source range MPEG (i.e. "video", 16-235)
247 1 -> source range JPEG (i.e. "full", 0-255)
249 0 -> destination range MPEG (i.e. "video", 16-235)
250 1 -> destination range JPEG (i.e. "full", 0-255)
252 But remember: sws_setColorspaceDetails ignores these
253 parameters unless the both source and destination images
254 are isYUV or isGray. (If either is not, it uses video range).
256 sws_setColorspaceDetails (
258 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), video_range == VideoRange::VIDEO ? 0 : 1,
259 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), out_video_range == VideoRange::VIDEO ? 0 : 1,
263 /* Prepare input data pointers with crop */
264 uint8_t* scale_in_data[planes()];
265 for (int c = 0; c < planes(); ++c) {
266 int const x = lrintf(bytes_per_pixel(c) * corrected_crop.left);
267 scale_in_data[c] = data()[c] + x + stride()[c] * (corrected_crop.top / vertical_factor(c));
270 auto out_desc = av_pix_fmt_desc_get (out_format);
272 throw PixelFormatError ("crop_scale_window()", out_format);
275 /* Corner of the image within out_size */
276 Position<int> const corner (
277 round_width_for_subsampling((out_size.width - inter_size.width) / 2, out_desc),
278 round_height_for_subsampling((out_size.height - inter_size.height) / 2, out_desc)
281 uint8_t* scale_out_data[out->planes()];
282 for (int c = 0; c < out->planes(); ++c) {
283 int const x = lrintf(out->bytes_per_pixel(c) * corner.x);
284 scale_out_data[c] = out->data()[c] + x + out->stride()[c] * (corner.y / out->vertical_factor(c));
289 scale_in_data, stride(),
290 0, cropped_size.height,
291 scale_out_data, out->stride()
294 sws_freeContext (scale_context);
296 if (corrected_crop != Crop() && cropped_size == inter_size) {
297 /* We are cropping without any scaling or pixel format conversion, so FFmpeg may have left some
298 data behind in our image. Clear it out. It may get to the point where we should just stop
299 trying to be clever with cropping.
301 out->make_part_black (corner.x + cropped_size.width, out_size.width - cropped_size.width);
305 video_range == VideoRange::VIDEO &&
306 out_video_range == VideoRange::FULL &&
307 av_pix_fmt_desc_get(_pixel_format)->flags & AV_PIX_FMT_FLAG_RGB
309 /* libswscale will not convert video range for RGB sources, so we have to do it ourselves */
310 out->video_range_to_full_range ();
318 Image::convert_pixel_format (dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, Alignment out_alignment, bool fast) const
320 return scale(size(), yuv_to_rgb, out_format, out_alignment, fast);
324 /** @param out_size Size to scale to.
325 * @param yuv_to_rgb YUVToRGB transform transform to use, if required.
326 * @param out_format Output pixel format.
327 * @param out_aligment Output alignment.
328 * @param fast Try to be fast at the possible expense of quality; at present this means using
329 * fast bilinear rather than bicubic scaling.
332 Image::scale (dcp::Size out_size, dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, Alignment out_alignment, bool fast) const
334 /* Empirical testing suggests that sws_scale() will crash if
335 the input image alignment is not PADDED.
337 DCPOMATIC_ASSERT (alignment() == Alignment::PADDED);
339 auto scaled = make_shared<Image>(out_format, out_size, out_alignment);
340 auto scale_context = sws_getContext (
341 size().width, size().height, pixel_format(),
342 out_size.width, out_size.height, out_format,
343 (fast ? SWS_FAST_BILINEAR : SWS_BICUBIC) | SWS_ACCURATE_RND, 0, 0, 0
346 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
347 int const lut[static_cast<int>(dcp::YUVToRGB::COUNT)] = {
352 /* The 3rd parameter here is:
353 0 -> source range MPEG (i.e. "video", 16-235)
354 1 -> source range JPEG (i.e. "full", 0-255)
356 0 -> destination range MPEG (i.e. "video", 16-235)
357 1 -> destination range JPEG (i.e. "full", 0-255)
359 But remember: sws_setColorspaceDetails ignores these
360 parameters unless the corresponding image isYUV or isGray.
361 (If it's neither, it uses video range).
363 sws_setColorspaceDetails (
365 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), 0,
366 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), 0,
374 scaled->data(), scaled->stride()
377 sws_freeContext (scale_context);
383 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
385 Image::yuv_16_black (uint16_t v, bool alpha)
387 memset (data()[0], 0, sample_size(0).height * stride()[0]);
388 for (int i = 1; i < 3; ++i) {
389 auto p = reinterpret_cast<int16_t*> (data()[i]);
390 int const lines = sample_size(i).height;
391 for (int y = 0; y < lines; ++y) {
392 /* We divide by 2 here because we are writing 2 bytes at a time */
393 for (int x = 0; x < line_size()[i] / 2; ++x) {
396 p += stride()[i] / 2;
401 memset (data()[3], 0, sample_size(3).height * stride()[3]);
407 Image::swap_16 (uint16_t v)
409 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
414 Image::make_part_black (int const start, int const width)
416 auto y_part = [&]() {
417 int const bpp = bytes_per_pixel(0);
418 int const h = sample_size(0).height;
419 int const s = stride()[0];
421 for (int y = 0; y < h; ++y) {
422 memset (p + start * bpp, 0, width * bpp);
427 switch (_pixel_format) {
428 case AV_PIX_FMT_RGB24:
429 case AV_PIX_FMT_ARGB:
430 case AV_PIX_FMT_RGBA:
431 case AV_PIX_FMT_ABGR:
432 case AV_PIX_FMT_BGRA:
433 case AV_PIX_FMT_RGB555LE:
434 case AV_PIX_FMT_RGB48LE:
435 case AV_PIX_FMT_RGB48BE:
436 case AV_PIX_FMT_XYZ12LE:
438 int const h = sample_size(0).height;
439 int const bpp = bytes_per_pixel(0);
440 int const s = stride()[0];
441 uint8_t* p = data()[0];
442 for (int y = 0; y < h; y++) {
443 memset (p + start * bpp, 0, width * bpp);
448 case AV_PIX_FMT_YUV420P:
451 for (int i = 1; i < 3; ++i) {
453 int const h = sample_size(i).height;
454 for (int y = 0; y < h; ++y) {
455 for (int x = start / 2; x < (start + width) / 2; ++x) {
463 case AV_PIX_FMT_YUV422P10LE:
466 for (int i = 1; i < 3; ++i) {
467 auto p = reinterpret_cast<int16_t*>(data()[i]);
468 int const h = sample_size(i).height;
469 for (int y = 0; y < h; ++y) {
470 for (int x = start / 2; x < (start + width) / 2; ++x) {
473 p += stride()[i] / 2;
479 throw PixelFormatError ("make_part_black()", _pixel_format);
487 switch (_pixel_format) {
488 case AV_PIX_FMT_YUV420P:
489 case AV_PIX_FMT_YUV422P:
490 case AV_PIX_FMT_YUV444P:
491 case AV_PIX_FMT_YUV411P:
492 memset (data()[0], 0, sample_size(0).height * stride()[0]);
493 memset (data()[1], eight_bit_uv, sample_size(1).height * stride()[1]);
494 memset (data()[2], eight_bit_uv, sample_size(2).height * stride()[2]);
497 case AV_PIX_FMT_YUVJ420P:
498 case AV_PIX_FMT_YUVJ422P:
499 case AV_PIX_FMT_YUVJ444P:
500 memset (data()[0], 0, sample_size(0).height * stride()[0]);
501 memset (data()[1], eight_bit_uv + 1, sample_size(1).height * stride()[1]);
502 memset (data()[2], eight_bit_uv + 1, sample_size(2).height * stride()[2]);
505 case AV_PIX_FMT_YUV422P9LE:
506 case AV_PIX_FMT_YUV444P9LE:
507 yuv_16_black (nine_bit_uv, false);
510 case AV_PIX_FMT_YUV422P9BE:
511 case AV_PIX_FMT_YUV444P9BE:
512 yuv_16_black (swap_16 (nine_bit_uv), false);
515 case AV_PIX_FMT_YUV422P10LE:
516 case AV_PIX_FMT_YUV444P10LE:
517 yuv_16_black (ten_bit_uv, false);
520 case AV_PIX_FMT_YUV422P16LE:
521 case AV_PIX_FMT_YUV444P16LE:
522 yuv_16_black (sixteen_bit_uv, false);
525 case AV_PIX_FMT_YUV444P10BE:
526 case AV_PIX_FMT_YUV422P10BE:
527 yuv_16_black (swap_16 (ten_bit_uv), false);
530 case AV_PIX_FMT_YUVA420P9BE:
531 case AV_PIX_FMT_YUVA422P9BE:
532 case AV_PIX_FMT_YUVA444P9BE:
533 yuv_16_black (swap_16 (nine_bit_uv), true);
536 case AV_PIX_FMT_YUVA420P9LE:
537 case AV_PIX_FMT_YUVA422P9LE:
538 case AV_PIX_FMT_YUVA444P9LE:
539 yuv_16_black (nine_bit_uv, true);
542 case AV_PIX_FMT_YUVA420P10BE:
543 case AV_PIX_FMT_YUVA422P10BE:
544 case AV_PIX_FMT_YUVA444P10BE:
545 yuv_16_black (swap_16 (ten_bit_uv), true);
548 case AV_PIX_FMT_YUVA420P10LE:
549 case AV_PIX_FMT_YUVA422P10LE:
550 case AV_PIX_FMT_YUVA444P10LE:
551 yuv_16_black (ten_bit_uv, true);
554 case AV_PIX_FMT_YUVA420P16BE:
555 case AV_PIX_FMT_YUVA422P16BE:
556 case AV_PIX_FMT_YUVA444P16BE:
557 yuv_16_black (swap_16 (sixteen_bit_uv), true);
560 case AV_PIX_FMT_YUVA420P16LE:
561 case AV_PIX_FMT_YUVA422P16LE:
562 case AV_PIX_FMT_YUVA444P16LE:
563 yuv_16_black (sixteen_bit_uv, true);
566 case AV_PIX_FMT_RGB24:
567 case AV_PIX_FMT_ARGB:
568 case AV_PIX_FMT_RGBA:
569 case AV_PIX_FMT_ABGR:
570 case AV_PIX_FMT_BGRA:
571 case AV_PIX_FMT_RGB555LE:
572 case AV_PIX_FMT_RGB48LE:
573 case AV_PIX_FMT_RGB48BE:
574 case AV_PIX_FMT_XYZ12LE:
575 memset (data()[0], 0, sample_size(0).height * stride()[0]);
578 case AV_PIX_FMT_UYVY422:
580 int const Y = sample_size(0).height;
581 int const X = line_size()[0];
582 uint8_t* p = data()[0];
583 for (int y = 0; y < Y; ++y) {
584 for (int x = 0; x < X / 4; ++x) {
585 *p++ = eight_bit_uv; // Cb
587 *p++ = eight_bit_uv; // Cr
595 throw PixelFormatError ("make_black()", _pixel_format);
601 Image::make_transparent ()
603 if (_pixel_format != AV_PIX_FMT_BGRA && _pixel_format != AV_PIX_FMT_RGBA) {
604 throw PixelFormatError ("make_transparent()", _pixel_format);
607 memset (data()[0], 0, sample_size(0).height * stride()[0]);
612 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
614 /* We're blending RGBA or BGRA images */
615 DCPOMATIC_ASSERT (other->pixel_format() == AV_PIX_FMT_BGRA || other->pixel_format() == AV_PIX_FMT_RGBA);
616 int const blue = other->pixel_format() == AV_PIX_FMT_BGRA ? 0 : 2;
617 int const red = other->pixel_format() == AV_PIX_FMT_BGRA ? 2 : 0;
619 int const other_bpp = 4;
621 int start_tx = position.x;
625 start_ox = -start_tx;
629 int start_ty = position.y;
633 start_oy = -start_ty;
637 switch (_pixel_format) {
638 case AV_PIX_FMT_RGB24:
640 /* Going onto RGB24. First byte is red, second green, third blue */
641 int const this_bpp = 3;
642 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
643 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
644 uint8_t* op = other->data()[0] + oy * other->stride()[0];
645 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
646 float const alpha = float (op[3]) / 255;
647 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
648 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
649 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
657 case AV_PIX_FMT_BGRA:
659 int const this_bpp = 4;
660 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
661 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
662 uint8_t* op = other->data()[0] + oy * other->stride()[0];
663 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
664 float const alpha = float (op[3]) / 255;
665 tp[0] = op[blue] * alpha + tp[0] * (1 - alpha);
666 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
667 tp[2] = op[red] * alpha + tp[2] * (1 - alpha);
668 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
676 case AV_PIX_FMT_RGBA:
678 int const this_bpp = 4;
679 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
680 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
681 uint8_t* op = other->data()[0] + oy * other->stride()[0];
682 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
683 float const alpha = float (op[3]) / 255;
684 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
685 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
686 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
687 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
695 case AV_PIX_FMT_RGB48LE:
697 int const this_bpp = 6;
698 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
699 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
700 uint8_t* op = other->data()[0] + oy * other->stride()[0];
701 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
702 float const alpha = float (op[3]) / 255;
703 /* Blend high bytes */
704 tp[1] = op[red] * alpha + tp[1] * (1 - alpha);
705 tp[3] = op[1] * alpha + tp[3] * (1 - alpha);
706 tp[5] = op[blue] * alpha + tp[5] * (1 - alpha);
714 case AV_PIX_FMT_XYZ12LE:
716 auto conv = dcp::ColourConversion::srgb_to_xyz();
717 double fast_matrix[9];
718 dcp::combined_rgb_to_xyz (conv, fast_matrix);
719 double const * lut_in = conv.in()->lut (8, false);
720 double const * lut_out = conv.out()->lut (16, true);
721 int const this_bpp = 6;
722 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
723 uint16_t* tp = reinterpret_cast<uint16_t*> (data()[0] + ty * stride()[0] + start_tx * this_bpp);
724 uint8_t* op = other->data()[0] + oy * other->stride()[0];
725 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
726 float const alpha = float (op[3]) / 255;
728 /* Convert sRGB to XYZ; op is BGRA. First, input gamma LUT */
729 double const r = lut_in[op[red]];
730 double const g = lut_in[op[1]];
731 double const b = lut_in[op[blue]];
733 /* RGB to XYZ, including Bradford transform and DCI companding */
734 double const x = max (0.0, min (65535.0, r * fast_matrix[0] + g * fast_matrix[1] + b * fast_matrix[2]));
735 double const y = max (0.0, min (65535.0, r * fast_matrix[3] + g * fast_matrix[4] + b * fast_matrix[5]));
736 double const z = max (0.0, min (65535.0, r * fast_matrix[6] + g * fast_matrix[7] + b * fast_matrix[8]));
738 /* Out gamma LUT and blend */
739 tp[0] = lrint(lut_out[lrint(x)] * 65535) * alpha + tp[0] * (1 - alpha);
740 tp[1] = lrint(lut_out[lrint(y)] * 65535) * alpha + tp[1] * (1 - alpha);
741 tp[2] = lrint(lut_out[lrint(z)] * 65535) * alpha + tp[2] * (1 - alpha);
749 case AV_PIX_FMT_YUV420P:
751 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
752 dcp::Size const ts = size();
753 dcp::Size const os = yuv->size();
754 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
755 int const hty = ty / 2;
756 int const hoy = oy / 2;
757 uint8_t* tY = data()[0] + (ty * stride()[0]) + start_tx;
758 uint8_t* tU = data()[1] + (hty * stride()[1]) + start_tx / 2;
759 uint8_t* tV = data()[2] + (hty * stride()[2]) + start_tx / 2;
760 uint8_t* oY = yuv->data()[0] + (oy * yuv->stride()[0]) + start_ox;
761 uint8_t* oU = yuv->data()[1] + (hoy * yuv->stride()[1]) + start_ox / 2;
762 uint8_t* oV = yuv->data()[2] + (hoy * yuv->stride()[2]) + start_ox / 2;
763 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
764 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
765 float const a = float(alpha[3]) / 255;
766 *tY = *oY * a + *tY * (1 - a);
767 *tU = *oU * a + *tU * (1 - a);
768 *tV = *oV * a + *tV * (1 - a);
784 case AV_PIX_FMT_YUV420P10:
786 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
787 dcp::Size const ts = size();
788 dcp::Size const os = yuv->size();
789 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
790 int const hty = ty / 2;
791 int const hoy = oy / 2;
792 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
793 uint16_t* tU = ((uint16_t *) (data()[1] + (hty * stride()[1]))) + start_tx / 2;
794 uint16_t* tV = ((uint16_t *) (data()[2] + (hty * stride()[2]))) + start_tx / 2;
795 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
796 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (hoy * yuv->stride()[1]))) + start_ox / 2;
797 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (hoy * yuv->stride()[2]))) + start_ox / 2;
798 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
799 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
800 float const a = float(alpha[3]) / 255;
801 *tY = *oY * a + *tY * (1 - a);
802 *tU = *oU * a + *tU * (1 - a);
803 *tV = *oV * a + *tV * (1 - a);
819 case AV_PIX_FMT_YUV422P10LE:
821 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
822 dcp::Size const ts = size();
823 dcp::Size const os = yuv->size();
824 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
825 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
826 uint16_t* tU = ((uint16_t *) (data()[1] + (ty * stride()[1]))) + start_tx / 2;
827 uint16_t* tV = ((uint16_t *) (data()[2] + (ty * stride()[2]))) + start_tx / 2;
828 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
829 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (oy * yuv->stride()[1]))) + start_ox / 2;
830 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (oy * yuv->stride()[2]))) + start_ox / 2;
831 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
832 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
833 float const a = float(alpha[3]) / 255;
834 *tY = *oY * a + *tY * (1 - a);
835 *tU = *oU * a + *tU * (1 - a);
836 *tV = *oV * a + *tV * (1 - a);
853 throw PixelFormatError ("alpha_blend()", _pixel_format);
859 Image::copy (shared_ptr<const Image> other, Position<int> position)
861 /* Only implemented for RGB24 onto RGB24 so far */
862 DCPOMATIC_ASSERT (_pixel_format == AV_PIX_FMT_RGB24 && other->pixel_format() == AV_PIX_FMT_RGB24);
863 DCPOMATIC_ASSERT (position.x >= 0 && position.y >= 0);
865 int const N = min (position.x + other->size().width, size().width) - position.x;
866 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
867 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
868 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
869 memcpy (tp, op, N * 3);
875 Image::read_from_socket (shared_ptr<Socket> socket)
877 for (int i = 0; i < planes(); ++i) {
878 uint8_t* p = data()[i];
879 int const lines = sample_size(i).height;
880 for (int y = 0; y < lines; ++y) {
881 socket->read (p, line_size()[i]);
889 Image::write_to_socket (shared_ptr<Socket> socket) const
891 for (int i = 0; i < planes(); ++i) {
892 uint8_t* p = data()[i];
893 int const lines = sample_size(i).height;
894 for (int y = 0; y < lines; ++y) {
895 socket->write (p, line_size()[i]);
903 Image::bytes_per_pixel (int c) const
905 auto d = av_pix_fmt_desc_get(_pixel_format);
907 throw PixelFormatError ("bytes_per_pixel()", _pixel_format);
914 float bpp[4] = { 0, 0, 0, 0 };
916 #ifdef DCPOMATIC_HAVE_AVCOMPONENTDESCRIPTOR_DEPTH_MINUS1
917 bpp[0] = floor ((d->comp[0].depth_minus1 + 8) / 8);
918 if (d->nb_components > 1) {
919 bpp[1] = floor ((d->comp[1].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
921 if (d->nb_components > 2) {
922 bpp[2] = floor ((d->comp[2].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
924 if (d->nb_components > 3) {
925 bpp[3] = floor ((d->comp[3].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
928 bpp[0] = floor ((d->comp[0].depth + 7) / 8);
929 if (d->nb_components > 1) {
930 bpp[1] = floor ((d->comp[1].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
932 if (d->nb_components > 2) {
933 bpp[2] = floor ((d->comp[2].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
935 if (d->nb_components > 3) {
936 bpp[3] = floor ((d->comp[3].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
940 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
941 /* Not planar; sum them up */
942 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
949 /** Construct a Image of a given size and format, allocating memory
952 * @param p Pixel format.
953 * @param s Size in pixels.
954 * @param alignment PADDED to make each row of this image aligned to a ALIGNMENT-byte boundary, otherwise COMPACT.
956 Image::Image (AVPixelFormat p, dcp::Size s, Alignment alignment)
959 , _alignment (alignment)
968 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
969 _data[0] = _data[1] = _data[2] = _data[3] = 0;
971 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
972 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
974 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
975 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
977 auto stride_round_up = [](int stride, int t) {
978 int const a = stride + (t - 1);
982 for (int i = 0; i < planes(); ++i) {
983 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
984 _stride[i] = stride_round_up (_line_size[i], _alignment == Alignment::PADDED ? ALIGNMENT : 1);
986 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
987 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
988 Hence on the last pixel of the last line it reads over the end of
989 the actual data by 1 byte. If the width of an image is a multiple
990 of the stride alignment there will be no padding at the end of image lines.
991 OS X crashes on this illegal read, though other operating systems don't
992 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
993 for that instruction to read safely.
995 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
996 over-reads by more then _avx. I can't follow the code to work out how much,
997 so I'll just over-allocate by ALIGNMENT bytes and have done with it. Empirical
998 testing suggests that it works.
1000 In addition to these concerns, we may read/write as much as a whole extra line
1001 at the end of each plane in cases where we are messing with offsets in order to
1002 do pad or crop. To solve this we over-allocate by an extra _stride[i] bytes.
1004 As an example: we may write to images starting at an offset so we get some padding.
1005 Hence we want to write in the following pattern:
1007 block start write start line end
1008 |..(padding)..|<------line-size------------->|..(padding)..|
1009 |..(padding)..|<------line-size------------->|..(padding)..|
1010 |..(padding)..|<------line-size------------->|..(padding)..|
1012 where line-size is of the smaller (inter_size) image and the full padded line length is that of
1013 out_size. To get things to work we have to tell FFmpeg that the stride is that of out_size.
1014 However some parts of FFmpeg (notably rgb48Toxyz12 in swscale.c) process data for the full
1015 specified *stride*. This does not matter until we get to the last line:
1017 block start write start line end
1018 |..(padding)..|<------line-size------------->|XXXwrittenXXX|
1019 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXX|
1020 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXXXXXwrittenXXX
1023 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
1024 #if HAVE_VALGRIND_MEMCHECK_H
1025 /* The data between the end of the line size and the stride is undefined but processed by
1026 libswscale, causing lots of valgrind errors. Mark it all defined to quell these errors.
1028 VALGRIND_MAKE_MEM_DEFINED (_data[i], _stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
1034 Image::Image (Image const & other)
1035 : std::enable_shared_from_this<Image>(other)
1036 , _size (other._size)
1037 , _pixel_format (other._pixel_format)
1038 , _alignment (other._alignment)
1042 for (int i = 0; i < planes(); ++i) {
1043 uint8_t* p = _data[i];
1044 uint8_t* q = other._data[i];
1045 int const lines = sample_size(i).height;
1046 for (int j = 0; j < lines; ++j) {
1047 memcpy (p, q, _line_size[i]);
1049 q += other.stride()[i];
1055 Image::Image (AVFrame const * frame, Alignment alignment)
1056 : _size (frame->width, frame->height)
1057 , _pixel_format (static_cast<AVPixelFormat>(frame->format))
1058 , _alignment (alignment)
1060 DCPOMATIC_ASSERT (_pixel_format != AV_PIX_FMT_NONE);
1064 for (int i = 0; i < planes(); ++i) {
1065 uint8_t* p = _data[i];
1066 uint8_t* q = frame->data[i];
1067 int const lines = sample_size(i).height;
1068 for (int j = 0; j < lines; ++j) {
1069 memcpy (p, q, _line_size[i]);
1071 /* AVFrame's linesize is what we call `stride' */
1072 q += frame->linesize[i];
1078 Image::Image (shared_ptr<const Image> other, Alignment alignment)
1079 : _size (other->_size)
1080 , _pixel_format (other->_pixel_format)
1081 , _alignment (alignment)
1085 for (int i = 0; i < planes(); ++i) {
1086 DCPOMATIC_ASSERT (line_size()[i] == other->line_size()[i]);
1087 uint8_t* p = _data[i];
1088 uint8_t* q = other->data()[i];
1089 int const lines = sample_size(i).height;
1090 for (int j = 0; j < lines; ++j) {
1091 memcpy (p, q, line_size()[i]);
1093 q += other->stride()[i];
1100 Image::operator= (Image const & other)
1102 if (this == &other) {
1113 Image::swap (Image & other)
1115 std::swap (_size, other._size);
1116 std::swap (_pixel_format, other._pixel_format);
1118 for (int i = 0; i < 4; ++i) {
1119 std::swap (_data[i], other._data[i]);
1120 std::swap (_line_size[i], other._line_size[i]);
1121 std::swap (_stride[i], other._stride[i]);
1124 std::swap (_alignment, other._alignment);
1130 for (int i = 0; i < planes(); ++i) {
1135 av_free (_line_size);
1141 Image::data () const
1148 Image::line_size () const
1155 Image::stride () const
1162 Image::size () const
1169 Image::alignment () const
1176 merge (list<PositionImage> images, Image::Alignment alignment)
1178 if (images.empty ()) {
1182 if (images.size() == 1) {
1183 images.front().image = Image::ensure_alignment(images.front().image, alignment);
1184 return images.front();
1187 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
1188 for (auto const& i: images) {
1189 all.extend (dcpomatic::Rect<int>(i.position, i.image->size().width, i.image->size().height));
1192 auto merged = make_shared<Image>(images.front().image->pixel_format(), dcp::Size(all.width, all.height), alignment);
1193 merged->make_transparent ();
1194 for (auto const& i: images) {
1195 merged->alpha_blend (i.image, i.position - all.position());
1198 return PositionImage (merged, all.position ());
1203 operator== (Image const & a, Image const & b)
1205 if (a.planes() != b.planes() || a.pixel_format() != b.pixel_format() || a.alignment() != b.alignment()) {
1209 for (int c = 0; c < a.planes(); ++c) {
1210 if (a.sample_size(c).height != b.sample_size(c).height || a.line_size()[c] != b.line_size()[c] || a.stride()[c] != b.stride()[c]) {
1214 uint8_t* p = a.data()[c];
1215 uint8_t* q = b.data()[c];
1216 int const lines = a.sample_size(c).height;
1217 for (int y = 0; y < lines; ++y) {
1218 if (memcmp (p, q, a.line_size()[c]) != 0) {
1232 * @param f Amount to fade by; 0 is black, 1 is no fade.
1235 Image::fade (float f)
1237 /* U/V black value for 8-bit colour */
1238 static int const eight_bit_uv = (1 << 7) - 1;
1239 /* U/V black value for 10-bit colour */
1240 static uint16_t const ten_bit_uv = (1 << 9) - 1;
1242 switch (_pixel_format) {
1243 case AV_PIX_FMT_YUV420P:
1246 uint8_t* p = data()[0];
1247 int const lines = sample_size(0).height;
1248 for (int y = 0; y < lines; ++y) {
1250 for (int x = 0; x < line_size()[0]; ++x) {
1251 *q = int(float(*q) * f);
1258 for (int c = 1; c < 3; ++c) {
1259 uint8_t* p = data()[c];
1260 int const lines = sample_size(c).height;
1261 for (int y = 0; y < lines; ++y) {
1263 for (int x = 0; x < line_size()[c]; ++x) {
1264 *q = eight_bit_uv + int((int(*q) - eight_bit_uv) * f);
1274 case AV_PIX_FMT_RGB24:
1277 uint8_t* p = data()[0];
1278 int const lines = sample_size(0).height;
1279 for (int y = 0; y < lines; ++y) {
1281 for (int x = 0; x < line_size()[0]; ++x) {
1282 *q = int (float (*q) * f);
1290 case AV_PIX_FMT_XYZ12LE:
1291 case AV_PIX_FMT_RGB48LE:
1292 /* 16-bit little-endian */
1293 for (int c = 0; c < 3; ++c) {
1294 int const stride_pixels = stride()[c] / 2;
1295 int const line_size_pixels = line_size()[c] / 2;
1296 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1297 int const lines = sample_size(c).height;
1298 for (int y = 0; y < lines; ++y) {
1300 for (int x = 0; x < line_size_pixels; ++x) {
1301 *q = int (float (*q) * f);
1309 case AV_PIX_FMT_YUV422P10LE:
1313 int const stride_pixels = stride()[0] / 2;
1314 int const line_size_pixels = line_size()[0] / 2;
1315 uint16_t* p = reinterpret_cast<uint16_t*> (data()[0]);
1316 int const lines = sample_size(0).height;
1317 for (int y = 0; y < lines; ++y) {
1319 for (int x = 0; x < line_size_pixels; ++x) {
1320 *q = int(float(*q) * f);
1328 for (int c = 1; c < 3; ++c) {
1329 int const stride_pixels = stride()[c] / 2;
1330 int const line_size_pixels = line_size()[c] / 2;
1331 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1332 int const lines = sample_size(c).height;
1333 for (int y = 0; y < lines; ++y) {
1335 for (int x = 0; x < line_size_pixels; ++x) {
1336 *q = ten_bit_uv + int((int(*q) - ten_bit_uv) * f);
1347 throw PixelFormatError ("fade()", _pixel_format);
1352 shared_ptr<const Image>
1353 Image::ensure_alignment (shared_ptr<const Image> image, Image::Alignment alignment)
1355 if (image->alignment() == alignment) {
1359 return make_shared<Image>(image, alignment);
1364 Image::memory_used () const
1367 for (int i = 0; i < planes(); ++i) {
1368 m += _stride[i] * sample_size(i).height;
1393 png_write_data (png_structp png_ptr, png_bytep data, png_size_t length)
1395 auto mem = reinterpret_cast<Memory*>(png_get_io_ptr(png_ptr));
1396 size_t size = mem->size + length;
1399 mem->data = reinterpret_cast<uint8_t*>(realloc(mem->data, size));
1401 mem->data = reinterpret_cast<uint8_t*>(malloc(size));
1405 throw EncodeError (N_("could not allocate memory for PNG"));
1408 memcpy (mem->data + mem->size, data, length);
1409 mem->size += length;
1414 png_flush (png_structp)
1421 png_error_fn (png_structp png_ptr, char const * message)
1423 reinterpret_cast<Image*>(png_get_error_ptr(png_ptr))->png_error (message);
1428 Image::png_error (char const * message)
1430 throw EncodeError (String::compose ("Error during PNG write: %1", message));
1435 Image::as_png () const
1437 DCPOMATIC_ASSERT (bytes_per_pixel(0) == 4);
1438 DCPOMATIC_ASSERT (planes() == 1);
1439 if (pixel_format() != AV_PIX_FMT_RGBA) {
1440 return convert_pixel_format(dcp::YUVToRGB::REC709, AV_PIX_FMT_RGBA, Image::Alignment::PADDED, false)->as_png();
1443 /* error handling? */
1444 png_structp png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, reinterpret_cast<void*>(const_cast<Image*>(this)), png_error_fn, 0);
1446 throw EncodeError (N_("could not create PNG write struct"));
1451 png_set_write_fn (png_ptr, &state, png_write_data, png_flush);
1453 png_infop info_ptr = png_create_info_struct(png_ptr);
1455 png_destroy_write_struct (&png_ptr, &info_ptr);
1456 throw EncodeError (N_("could not create PNG info struct"));
1459 png_set_IHDR (png_ptr, info_ptr, size().width, size().height, 8, PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT);
1461 png_byte ** row_pointers = reinterpret_cast<png_byte **>(png_malloc(png_ptr, size().height * sizeof(png_byte *)));
1462 for (int i = 0; i < size().height; ++i) {
1463 row_pointers[i] = (png_byte *) (data()[0] + i * stride()[0]);
1466 png_write_info (png_ptr, info_ptr);
1467 png_write_image (png_ptr, row_pointers);
1468 png_write_end (png_ptr, info_ptr);
1470 png_destroy_write_struct (&png_ptr, &info_ptr);
1471 png_free (png_ptr, row_pointers);
1473 return dcp::ArrayData (state.data, state.size);
1478 Image::video_range_to_full_range ()
1480 switch (_pixel_format) {
1481 case AV_PIX_FMT_RGB24:
1483 float const factor = 256.0 / 219.0;
1484 uint8_t* p = data()[0];
1485 int const lines = sample_size(0).height;
1486 for (int y = 0; y < lines; ++y) {
1488 for (int x = 0; x < line_size()[0]; ++x) {
1489 *q = clamp(lrintf((*q - 16) * factor), 0L, 255L);
1496 case AV_PIX_FMT_RGB48LE:
1498 float const factor = 65536.0 / 56064.0;
1499 uint16_t* p = reinterpret_cast<uint16_t*>(data()[0]);
1500 int const lines = sample_size(0).height;
1501 for (int y = 0; y < lines; ++y) {
1503 int const line_size_pixels = line_size()[0] / 2;
1504 for (int x = 0; x < line_size_pixels; ++x) {
1505 *q = clamp(lrintf((*q - 4096) * factor), 0L, 65535L);
1508 p += stride()[0] / 2;
1512 case AV_PIX_FMT_GBRP12LE:
1514 float const factor = 4096.0 / 3504.0;
1515 for (int c = 0; c < 3; ++c) {
1516 uint16_t* p = reinterpret_cast<uint16_t*>(data()[c]);
1517 int const lines = sample_size(c).height;
1518 for (int y = 0; y < lines; ++y) {
1520 int const line_size_pixels = line_size()[c] / 2;
1521 for (int x = 0; x < line_size_pixels; ++x) {
1522 *q = clamp(lrintf((*q - 256) * factor), 0L, 4095L);
1530 throw PixelFormatError ("video_range_to_full_range()", _pixel_format);