2 Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 /** @file src/image.cc
23 * @brief A class to describe a video image.
27 #include "compose.hpp"
28 #include "dcpomatic_socket.h"
29 #include "exceptions.h"
34 #include <dcp/rgb_xyz.h>
35 #include <dcp/transfer_function.h>
37 #include <libavutil/frame.h>
38 #include <libavutil/pixdesc.h>
39 #include <libavutil/pixfmt.h>
40 #include <libswscale/swscale.h>
43 #if HAVE_VALGRIND_MEMCHECK_H
44 #include <valgrind/memcheck.h>
55 using std::make_shared;
58 using std::runtime_error;
59 using std::shared_ptr;
64 /** The memory alignment, in bytes, used for each row of an image if aligment is requested */
67 /* U/V black value for 8-bit colour */
68 static uint8_t const eight_bit_uv = (1 << 7) - 1;
69 /* U/V black value for 9-bit colour */
70 static uint16_t const nine_bit_uv = (1 << 8) - 1;
71 /* U/V black value for 10-bit colour */
72 static uint16_t const ten_bit_uv = (1 << 9) - 1;
73 /* U/V black value for 16-bit colour */
74 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
78 Image::vertical_factor (int n) const
84 auto d = av_pix_fmt_desc_get(_pixel_format);
86 throw PixelFormatError ("line_factor()", _pixel_format);
89 return lrintf(powf(2.0f, d->log2_chroma_h));
93 Image::horizontal_factor (int n) const
99 auto d = av_pix_fmt_desc_get(_pixel_format);
101 throw PixelFormatError ("sample_size()", _pixel_format);
104 return lrintf(powf(2.0f, d->log2_chroma_w));
107 /** @param n Component index.
108 * @return Number of samples (i.e. pixels, unless sub-sampled) in each direction for this component.
111 Image::sample_size (int n) const
114 lrint (ceil(static_cast<double>(size().width) / horizontal_factor(n))),
115 lrint (ceil(static_cast<double>(size().height) / vertical_factor(n)))
119 /** @return Number of planes */
121 Image::planes () const
123 auto d = av_pix_fmt_desc_get(_pixel_format);
125 throw PixelFormatError ("planes()", _pixel_format);
128 if (_pixel_format == AV_PIX_FMT_PAL8) {
132 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
136 return d->nb_components;
142 round_width_for_subsampling (int p, AVPixFmtDescriptor const * desc)
144 return p & ~ ((1 << desc->log2_chroma_w) - 1);
150 round_height_for_subsampling (int p, AVPixFmtDescriptor const * desc)
152 return p & ~ ((1 << desc->log2_chroma_h) - 1);
156 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size'.
157 * @param crop Amount to crop by.
158 * @param inter_size Size to scale the cropped image to.
159 * @param out_size Size of output frame; if this is larger than inter_size there will be black padding.
160 * @param yuv_to_rgb YUV to RGB transformation to use, if required.
161 * @param video_range Video range of the image.
162 * @param out_format Output pixel format.
163 * @param out_aligned true to make the output image aligned.
164 * @param out_video_range Video range to use for the output image.
165 * @param fast Try to be fast at the possible expense of quality; at present this means using
166 * fast bilinear rather than bicubic scaling.
169 Image::crop_scale_window (
171 dcp::Size inter_size,
173 dcp::YUVToRGB yuv_to_rgb,
174 VideoRange video_range,
175 AVPixelFormat out_format,
176 VideoRange out_video_range,
181 /* Empirical testing suggests that sws_scale() will crash if
182 the input image is not aligned.
184 DCPOMATIC_ASSERT (aligned ());
186 DCPOMATIC_ASSERT (out_size.width >= inter_size.width);
187 DCPOMATIC_ASSERT (out_size.height >= inter_size.height);
189 auto out = make_shared<Image>(out_format, out_size, out_aligned);
192 auto in_desc = av_pix_fmt_desc_get (_pixel_format);
194 throw PixelFormatError ("crop_scale_window()", _pixel_format);
197 /* Round down so that we crop only the number of pixels that is straightforward
198 * considering any subsampling.
201 round_width_for_subsampling(crop.left, in_desc),
202 round_width_for_subsampling(crop.right, in_desc),
203 round_height_for_subsampling(crop.top, in_desc),
204 round_height_for_subsampling(crop.bottom, in_desc)
207 /* Also check that we aren't cropping more image than there actually is */
208 if ((corrected_crop.left + corrected_crop.right) >= (size().width - 4)) {
209 corrected_crop.left = 0;
210 corrected_crop.right = size().width - 4;
213 if ((corrected_crop.top + corrected_crop.bottom) >= (size().height - 4)) {
214 corrected_crop.top = 0;
215 corrected_crop.bottom = size().height - 4;
218 /* Size of the image after any crop */
219 auto const cropped_size = corrected_crop.apply (size());
221 /* Scale context for a scale from cropped_size to inter_size */
222 auto scale_context = sws_getContext (
223 cropped_size.width, cropped_size.height, pixel_format(),
224 inter_size.width, inter_size.height, out_format,
225 fast ? SWS_FAST_BILINEAR : SWS_BICUBIC, 0, 0, 0
228 if (!scale_context) {
229 throw runtime_error (N_("Could not allocate SwsContext"));
232 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
233 int const lut[static_cast<int>(dcp::YUVToRGB::COUNT)] = {
238 /* The 3rd parameter here is:
239 0 -> source range MPEG (i.e. "video", 16-235)
240 1 -> source range JPEG (i.e. "full", 0-255)
242 0 -> destination range MPEG (i.e. "video", 16-235)
243 1 -> destination range JPEG (i.e. "full", 0-255)
245 But remember: sws_setColorspaceDetails ignores these
246 parameters unless the both source and destination images
247 are isYUV or isGray. (If either is not, it uses video range).
249 sws_setColorspaceDetails (
251 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), video_range == VideoRange::VIDEO ? 0 : 1,
252 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), out_video_range == VideoRange::VIDEO ? 0 : 1,
256 /* Prepare input data pointers with crop */
257 uint8_t* scale_in_data[planes()];
258 for (int c = 0; c < planes(); ++c) {
259 int const x = lrintf(bytes_per_pixel(c) * corrected_crop.left);
260 scale_in_data[c] = data()[c] + x + stride()[c] * (corrected_crop.top / vertical_factor(c));
263 auto out_desc = av_pix_fmt_desc_get (out_format);
265 throw PixelFormatError ("crop_scale_window()", out_format);
268 /* Corner of the image within out_size */
269 Position<int> const corner (
270 round_width_for_subsampling((out_size.width - inter_size.width) / 2, out_desc),
271 round_height_for_subsampling((out_size.height - inter_size.height) / 2, out_desc)
274 uint8_t* scale_out_data[out->planes()];
275 for (int c = 0; c < out->planes(); ++c) {
276 int const x = lrintf(out->bytes_per_pixel(c) * corner.x);
277 scale_out_data[c] = out->data()[c] + x + out->stride()[c] * (corner.y / out->vertical_factor(c));
282 scale_in_data, stride(),
283 0, cropped_size.height,
284 scale_out_data, out->stride()
287 sws_freeContext (scale_context);
289 if (corrected_crop != Crop() && cropped_size == inter_size) {
290 /* We are cropping without any scaling or pixel format conversion, so FFmpeg may have left some
291 data behind in our image. Clear it out. It may get to the point where we should just stop
292 trying to be clever with cropping.
294 out->make_part_black (corner.x + cropped_size.width, out_size.width - cropped_size.width);
301 Image::convert_pixel_format (dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, bool out_aligned, bool fast) const
303 return scale(size(), yuv_to_rgb, out_format, out_aligned, fast);
306 /** @param out_size Size to scale to.
307 * @param yuv_to_rgb YUVToRGB transform transform to use, if required.
308 * @param out_format Output pixel format.
309 * @param out_aligned true to make an aligned output image.
310 * @param fast Try to be fast at the possible expense of quality; at present this means using
311 * fast bilinear rather than bicubic scaling.
314 Image::scale (dcp::Size out_size, dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, bool out_aligned, bool fast) const
316 /* Empirical testing suggests that sws_scale() will crash if
317 the input image is not aligned.
319 DCPOMATIC_ASSERT (aligned ());
321 auto scaled = make_shared<Image>(out_format, out_size, out_aligned);
322 auto scale_context = sws_getContext (
323 size().width, size().height, pixel_format(),
324 out_size.width, out_size.height, out_format,
325 (fast ? SWS_FAST_BILINEAR : SWS_BICUBIC) | SWS_ACCURATE_RND, 0, 0, 0
328 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
329 int const lut[static_cast<int>(dcp::YUVToRGB::COUNT)] = {
334 /* The 3rd parameter here is:
335 0 -> source range MPEG (i.e. "video", 16-235)
336 1 -> source range JPEG (i.e. "full", 0-255)
338 0 -> destination range MPEG (i.e. "video", 16-235)
339 1 -> destination range JPEG (i.e. "full", 0-255)
341 But remember: sws_setColorspaceDetails ignores these
342 parameters unless the corresponding image isYUV or isGray.
343 (If it's neither, it uses video range).
345 sws_setColorspaceDetails (
347 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), 0,
348 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), 0,
356 scaled->data(), scaled->stride()
359 sws_freeContext (scale_context);
364 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
366 Image::yuv_16_black (uint16_t v, bool alpha)
368 memset (data()[0], 0, sample_size(0).height * stride()[0]);
369 for (int i = 1; i < 3; ++i) {
370 auto p = reinterpret_cast<int16_t*> (data()[i]);
371 int const lines = sample_size(i).height;
372 for (int y = 0; y < lines; ++y) {
373 /* We divide by 2 here because we are writing 2 bytes at a time */
374 for (int x = 0; x < line_size()[i] / 2; ++x) {
377 p += stride()[i] / 2;
382 memset (data()[3], 0, sample_size(3).height * stride()[3]);
387 Image::swap_16 (uint16_t v)
389 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
393 Image::make_part_black (int const start, int const width)
395 switch (_pixel_format) {
396 case AV_PIX_FMT_RGB24:
397 case AV_PIX_FMT_ARGB:
398 case AV_PIX_FMT_RGBA:
399 case AV_PIX_FMT_ABGR:
400 case AV_PIX_FMT_BGRA:
401 case AV_PIX_FMT_RGB555LE:
402 case AV_PIX_FMT_RGB48LE:
403 case AV_PIX_FMT_RGB48BE:
404 case AV_PIX_FMT_XYZ12LE:
406 int const h = sample_size(0).height;
407 int const bpp = bytes_per_pixel(0);
408 int const s = stride()[0];
409 uint8_t* p = data()[0];
410 for (int y = 0; y < h; y++) {
411 memset (p + start * bpp, 0, width * bpp);
416 case AV_PIX_FMT_YUV422P10LE:
418 int const bpp_0 = bytes_per_pixel(0);
419 int const h_0 = sample_size(0).height;
420 int const stride_0 = stride()[0];
422 for (int y = 0; y < h_0; ++y) {
423 memset (p + start * bpp_0, 0xff, width * bpp_0);
426 for (int i = 1; i < 3; ++i) {
427 auto p = reinterpret_cast<int16_t*>(data()[i]);
428 int const lines = sample_size(i).height;
429 for (int y = 0; y < lines; ++y) {
430 for (int x = start / 2; x < (start + width) / 2; ++x) {
433 p += stride()[i] / 2;
439 throw PixelFormatError ("make_part_black()", _pixel_format);
446 switch (_pixel_format) {
447 case AV_PIX_FMT_YUV420P:
448 case AV_PIX_FMT_YUV422P:
449 case AV_PIX_FMT_YUV444P:
450 case AV_PIX_FMT_YUV411P:
451 memset (data()[0], 0, sample_size(0).height * stride()[0]);
452 memset (data()[1], eight_bit_uv, sample_size(1).height * stride()[1]);
453 memset (data()[2], eight_bit_uv, sample_size(2).height * stride()[2]);
456 case AV_PIX_FMT_YUVJ420P:
457 case AV_PIX_FMT_YUVJ422P:
458 case AV_PIX_FMT_YUVJ444P:
459 memset (data()[0], 0, sample_size(0).height * stride()[0]);
460 memset (data()[1], eight_bit_uv + 1, sample_size(1).height * stride()[1]);
461 memset (data()[2], eight_bit_uv + 1, sample_size(2).height * stride()[2]);
464 case AV_PIX_FMT_YUV422P9LE:
465 case AV_PIX_FMT_YUV444P9LE:
466 yuv_16_black (nine_bit_uv, false);
469 case AV_PIX_FMT_YUV422P9BE:
470 case AV_PIX_FMT_YUV444P9BE:
471 yuv_16_black (swap_16 (nine_bit_uv), false);
474 case AV_PIX_FMT_YUV422P10LE:
475 case AV_PIX_FMT_YUV444P10LE:
476 yuv_16_black (ten_bit_uv, false);
479 case AV_PIX_FMT_YUV422P16LE:
480 case AV_PIX_FMT_YUV444P16LE:
481 yuv_16_black (sixteen_bit_uv, false);
484 case AV_PIX_FMT_YUV444P10BE:
485 case AV_PIX_FMT_YUV422P10BE:
486 yuv_16_black (swap_16 (ten_bit_uv), false);
489 case AV_PIX_FMT_YUVA420P9BE:
490 case AV_PIX_FMT_YUVA422P9BE:
491 case AV_PIX_FMT_YUVA444P9BE:
492 yuv_16_black (swap_16 (nine_bit_uv), true);
495 case AV_PIX_FMT_YUVA420P9LE:
496 case AV_PIX_FMT_YUVA422P9LE:
497 case AV_PIX_FMT_YUVA444P9LE:
498 yuv_16_black (nine_bit_uv, true);
501 case AV_PIX_FMT_YUVA420P10BE:
502 case AV_PIX_FMT_YUVA422P10BE:
503 case AV_PIX_FMT_YUVA444P10BE:
504 yuv_16_black (swap_16 (ten_bit_uv), true);
507 case AV_PIX_FMT_YUVA420P10LE:
508 case AV_PIX_FMT_YUVA422P10LE:
509 case AV_PIX_FMT_YUVA444P10LE:
510 yuv_16_black (ten_bit_uv, true);
513 case AV_PIX_FMT_YUVA420P16BE:
514 case AV_PIX_FMT_YUVA422P16BE:
515 case AV_PIX_FMT_YUVA444P16BE:
516 yuv_16_black (swap_16 (sixteen_bit_uv), true);
519 case AV_PIX_FMT_YUVA420P16LE:
520 case AV_PIX_FMT_YUVA422P16LE:
521 case AV_PIX_FMT_YUVA444P16LE:
522 yuv_16_black (sixteen_bit_uv, true);
525 case AV_PIX_FMT_RGB24:
526 case AV_PIX_FMT_ARGB:
527 case AV_PIX_FMT_RGBA:
528 case AV_PIX_FMT_ABGR:
529 case AV_PIX_FMT_BGRA:
530 case AV_PIX_FMT_RGB555LE:
531 case AV_PIX_FMT_RGB48LE:
532 case AV_PIX_FMT_RGB48BE:
533 case AV_PIX_FMT_XYZ12LE:
534 memset (data()[0], 0, sample_size(0).height * stride()[0]);
537 case AV_PIX_FMT_UYVY422:
539 int const Y = sample_size(0).height;
540 int const X = line_size()[0];
541 uint8_t* p = data()[0];
542 for (int y = 0; y < Y; ++y) {
543 for (int x = 0; x < X / 4; ++x) {
544 *p++ = eight_bit_uv; // Cb
546 *p++ = eight_bit_uv; // Cr
554 throw PixelFormatError ("make_black()", _pixel_format);
559 Image::make_transparent ()
561 if (_pixel_format != AV_PIX_FMT_BGRA && _pixel_format != AV_PIX_FMT_RGBA) {
562 throw PixelFormatError ("make_transparent()", _pixel_format);
565 memset (data()[0], 0, sample_size(0).height * stride()[0]);
569 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
571 /* We're blending RGBA or BGRA images */
572 DCPOMATIC_ASSERT (other->pixel_format() == AV_PIX_FMT_BGRA || other->pixel_format() == AV_PIX_FMT_RGBA);
573 int const blue = other->pixel_format() == AV_PIX_FMT_BGRA ? 0 : 2;
574 int const red = other->pixel_format() == AV_PIX_FMT_BGRA ? 2 : 0;
576 int const other_bpp = 4;
578 int start_tx = position.x;
582 start_ox = -start_tx;
586 int start_ty = position.y;
590 start_oy = -start_ty;
594 switch (_pixel_format) {
595 case AV_PIX_FMT_RGB24:
597 /* Going onto RGB24. First byte is red, second green, third blue */
598 int const this_bpp = 3;
599 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
600 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
601 uint8_t* op = other->data()[0] + oy * other->stride()[0];
602 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
603 float const alpha = float (op[3]) / 255;
604 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
605 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
606 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
614 case AV_PIX_FMT_BGRA:
616 int const this_bpp = 4;
617 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
618 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
619 uint8_t* op = other->data()[0] + oy * other->stride()[0];
620 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
621 float const alpha = float (op[3]) / 255;
622 tp[0] = op[blue] * alpha + tp[0] * (1 - alpha);
623 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
624 tp[2] = op[red] * alpha + tp[2] * (1 - alpha);
625 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
633 case AV_PIX_FMT_RGBA:
635 int const this_bpp = 4;
636 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
637 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
638 uint8_t* op = other->data()[0] + oy * other->stride()[0];
639 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
640 float const alpha = float (op[3]) / 255;
641 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
642 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
643 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
644 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
652 case AV_PIX_FMT_RGB48LE:
654 int const this_bpp = 6;
655 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
656 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
657 uint8_t* op = other->data()[0] + oy * other->stride()[0];
658 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
659 float const alpha = float (op[3]) / 255;
660 /* Blend high bytes */
661 tp[1] = op[red] * alpha + tp[1] * (1 - alpha);
662 tp[3] = op[1] * alpha + tp[3] * (1 - alpha);
663 tp[5] = op[blue] * alpha + tp[5] * (1 - alpha);
671 case AV_PIX_FMT_XYZ12LE:
673 auto conv = dcp::ColourConversion::srgb_to_xyz();
674 double fast_matrix[9];
675 dcp::combined_rgb_to_xyz (conv, fast_matrix);
676 double const * lut_in = conv.in()->lut (8, false);
677 double const * lut_out = conv.out()->lut (16, true);
678 int const this_bpp = 6;
679 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
680 uint16_t* tp = reinterpret_cast<uint16_t*> (data()[0] + ty * stride()[0] + start_tx * this_bpp);
681 uint8_t* op = other->data()[0] + oy * other->stride()[0];
682 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
683 float const alpha = float (op[3]) / 255;
685 /* Convert sRGB to XYZ; op is BGRA. First, input gamma LUT */
686 double const r = lut_in[op[red]];
687 double const g = lut_in[op[1]];
688 double const b = lut_in[op[blue]];
690 /* RGB to XYZ, including Bradford transform and DCI companding */
691 double const x = max (0.0, min (65535.0, r * fast_matrix[0] + g * fast_matrix[1] + b * fast_matrix[2]));
692 double const y = max (0.0, min (65535.0, r * fast_matrix[3] + g * fast_matrix[4] + b * fast_matrix[5]));
693 double const z = max (0.0, min (65535.0, r * fast_matrix[6] + g * fast_matrix[7] + b * fast_matrix[8]));
695 /* Out gamma LUT and blend */
696 tp[0] = lrint(lut_out[lrint(x)] * 65535) * alpha + tp[0] * (1 - alpha);
697 tp[1] = lrint(lut_out[lrint(y)] * 65535) * alpha + tp[1] * (1 - alpha);
698 tp[2] = lrint(lut_out[lrint(z)] * 65535) * alpha + tp[2] * (1 - alpha);
706 case AV_PIX_FMT_YUV420P:
708 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, false, false);
709 dcp::Size const ts = size();
710 dcp::Size const os = yuv->size();
711 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
712 int const hty = ty / 2;
713 int const hoy = oy / 2;
714 uint8_t* tY = data()[0] + (ty * stride()[0]) + start_tx;
715 uint8_t* tU = data()[1] + (hty * stride()[1]) + start_tx / 2;
716 uint8_t* tV = data()[2] + (hty * stride()[2]) + start_tx / 2;
717 uint8_t* oY = yuv->data()[0] + (oy * yuv->stride()[0]) + start_ox;
718 uint8_t* oU = yuv->data()[1] + (hoy * yuv->stride()[1]) + start_ox / 2;
719 uint8_t* oV = yuv->data()[2] + (hoy * yuv->stride()[2]) + start_ox / 2;
720 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
721 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
722 float const a = float(alpha[3]) / 255;
723 *tY = *oY * a + *tY * (1 - a);
724 *tU = *oU * a + *tU * (1 - a);
725 *tV = *oV * a + *tV * (1 - a);
741 case AV_PIX_FMT_YUV420P10:
743 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, false, false);
744 dcp::Size const ts = size();
745 dcp::Size const os = yuv->size();
746 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
747 int const hty = ty / 2;
748 int const hoy = oy / 2;
749 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
750 uint16_t* tU = ((uint16_t *) (data()[1] + (hty * stride()[1]))) + start_tx / 2;
751 uint16_t* tV = ((uint16_t *) (data()[2] + (hty * stride()[2]))) + start_tx / 2;
752 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
753 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (hoy * yuv->stride()[1]))) + start_ox / 2;
754 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (hoy * yuv->stride()[2]))) + start_ox / 2;
755 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
756 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
757 float const a = float(alpha[3]) / 255;
758 *tY = *oY * a + *tY * (1 - a);
759 *tU = *oU * a + *tU * (1 - a);
760 *tV = *oV * a + *tV * (1 - a);
776 case AV_PIX_FMT_YUV422P10LE:
778 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, false, false);
779 dcp::Size const ts = size();
780 dcp::Size const os = yuv->size();
781 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
782 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
783 uint16_t* tU = ((uint16_t *) (data()[1] + (ty * stride()[1]))) + start_tx / 2;
784 uint16_t* tV = ((uint16_t *) (data()[2] + (ty * stride()[2]))) + start_tx / 2;
785 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
786 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (oy * yuv->stride()[1]))) + start_ox / 2;
787 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (oy * yuv->stride()[2]))) + start_ox / 2;
788 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
789 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
790 float const a = float(alpha[3]) / 255;
791 *tY = *oY * a + *tY * (1 - a);
792 *tU = *oU * a + *tU * (1 - a);
793 *tV = *oV * a + *tV * (1 - a);
810 throw PixelFormatError ("alpha_blend()", _pixel_format);
815 Image::copy (shared_ptr<const Image> other, Position<int> position)
817 /* Only implemented for RGB24 onto RGB24 so far */
818 DCPOMATIC_ASSERT (_pixel_format == AV_PIX_FMT_RGB24 && other->pixel_format() == AV_PIX_FMT_RGB24);
819 DCPOMATIC_ASSERT (position.x >= 0 && position.y >= 0);
821 int const N = min (position.x + other->size().width, size().width) - position.x;
822 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
823 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
824 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
825 memcpy (tp, op, N * 3);
830 Image::read_from_socket (shared_ptr<Socket> socket)
832 for (int i = 0; i < planes(); ++i) {
833 uint8_t* p = data()[i];
834 int const lines = sample_size(i).height;
835 for (int y = 0; y < lines; ++y) {
836 socket->read (p, line_size()[i]);
843 Image::write_to_socket (shared_ptr<Socket> socket) const
845 for (int i = 0; i < planes(); ++i) {
846 uint8_t* p = data()[i];
847 int const lines = sample_size(i).height;
848 for (int y = 0; y < lines; ++y) {
849 socket->write (p, line_size()[i]);
856 Image::bytes_per_pixel (int c) const
858 auto d = av_pix_fmt_desc_get(_pixel_format);
860 throw PixelFormatError ("bytes_per_pixel()", _pixel_format);
867 float bpp[4] = { 0, 0, 0, 0 };
869 #ifdef DCPOMATIC_HAVE_AVCOMPONENTDESCRIPTOR_DEPTH_MINUS1
870 bpp[0] = floor ((d->comp[0].depth_minus1 + 8) / 8);
871 if (d->nb_components > 1) {
872 bpp[1] = floor ((d->comp[1].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
874 if (d->nb_components > 2) {
875 bpp[2] = floor ((d->comp[2].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
877 if (d->nb_components > 3) {
878 bpp[3] = floor ((d->comp[3].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
881 bpp[0] = floor ((d->comp[0].depth + 7) / 8);
882 if (d->nb_components > 1) {
883 bpp[1] = floor ((d->comp[1].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
885 if (d->nb_components > 2) {
886 bpp[2] = floor ((d->comp[2].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
888 if (d->nb_components > 3) {
889 bpp[3] = floor ((d->comp[3].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
893 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
894 /* Not planar; sum them up */
895 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
901 /** Construct a Image of a given size and format, allocating memory
904 * @param p Pixel format.
905 * @param s Size in pixels.
906 * @param aligned true to make each row of this image aligned to a ALIGNMENT-byte boundary.
908 Image::Image (AVPixelFormat p, dcp::Size s, bool aligned)
919 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
920 _data[0] = _data[1] = _data[2] = _data[3] = 0;
922 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
923 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
925 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
926 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
928 for (int i = 0; i < planes(); ++i) {
929 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
930 _stride[i] = stride_round_up (i, _line_size, _aligned ? ALIGNMENT : 1);
932 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
933 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
934 Hence on the last pixel of the last line it reads over the end of
935 the actual data by 1 byte. If the width of an image is a multiple
936 of the stride alignment there will be no padding at the end of image lines.
937 OS X crashes on this illegal read, though other operating systems don't
938 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
939 for that instruction to read safely.
941 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
942 over-reads by more then _avx. I can't follow the code to work out how much,
943 so I'll just over-allocate by ALIGNMENT bytes and have done with it. Empirical
944 testing suggests that it works.
946 In addition to these concerns, we may read/write as much as a whole extra line
947 at the end of each plane in cases where we are messing with offsets in order to
948 do pad or crop. To solve this we over-allocate by an extra _stride[i] bytes.
950 As an example: we may write to images starting at an offset so we get some padding.
951 Hence we want to write in the following pattern:
953 block start write start line end
954 |..(padding)..|<------line-size------------->|..(padding)..|
955 |..(padding)..|<------line-size------------->|..(padding)..|
956 |..(padding)..|<------line-size------------->|..(padding)..|
958 where line-size is of the smaller (inter_size) image and the full padded line length is that of
959 out_size. To get things to work we have to tell FFmpeg that the stride is that of out_size.
960 However some parts of FFmpeg (notably rgb48Toxyz12 in swscale.c) process data for the full
961 specified *stride*. This does not matter until we get to the last line:
963 block start write start line end
964 |..(padding)..|<------line-size------------->|XXXwrittenXXX|
965 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXX|
966 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXXXXXwrittenXXX
969 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
970 #if HAVE_VALGRIND_MEMCHECK_H
971 /* The data between the end of the line size and the stride is undefined but processed by
972 libswscale, causing lots of valgrind errors. Mark it all defined to quell these errors.
974 VALGRIND_MAKE_MEM_DEFINED (_data[i], _stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
979 Image::Image (Image const & other)
980 : std::enable_shared_from_this<Image>(other)
981 , _size (other._size)
982 , _pixel_format (other._pixel_format)
983 , _aligned (other._aligned)
987 for (int i = 0; i < planes(); ++i) {
988 uint8_t* p = _data[i];
989 uint8_t* q = other._data[i];
990 int const lines = sample_size(i).height;
991 for (int j = 0; j < lines; ++j) {
992 memcpy (p, q, _line_size[i]);
994 q += other.stride()[i];
999 Image::Image (AVFrame* frame)
1000 : _size (frame->width, frame->height)
1001 , _pixel_format (static_cast<AVPixelFormat>(frame->format))
1004 DCPOMATIC_ASSERT (_pixel_format != AV_PIX_FMT_NONE);
1008 for (int i = 0; i < planes(); ++i) {
1009 uint8_t* p = _data[i];
1010 uint8_t* q = frame->data[i];
1011 int const lines = sample_size(i).height;
1012 for (int j = 0; j < lines; ++j) {
1013 memcpy (p, q, _line_size[i]);
1015 /* AVFrame's linesize is what we call `stride' */
1016 q += frame->linesize[i];
1021 Image::Image (shared_ptr<const Image> other, bool aligned)
1022 : _size (other->_size)
1023 , _pixel_format (other->_pixel_format)
1024 , _aligned (aligned)
1028 for (int i = 0; i < planes(); ++i) {
1029 DCPOMATIC_ASSERT (line_size()[i] == other->line_size()[i]);
1030 uint8_t* p = _data[i];
1031 uint8_t* q = other->data()[i];
1032 int const lines = sample_size(i).height;
1033 for (int j = 0; j < lines; ++j) {
1034 memcpy (p, q, line_size()[i]);
1036 q += other->stride()[i];
1042 Image::operator= (Image const & other)
1044 if (this == &other) {
1054 Image::swap (Image & other)
1056 std::swap (_size, other._size);
1057 std::swap (_pixel_format, other._pixel_format);
1059 for (int i = 0; i < 4; ++i) {
1060 std::swap (_data[i], other._data[i]);
1061 std::swap (_line_size[i], other._line_size[i]);
1062 std::swap (_stride[i], other._stride[i]);
1065 std::swap (_aligned, other._aligned);
1070 for (int i = 0; i < planes(); ++i) {
1075 av_free (_line_size);
1080 Image::data () const
1086 Image::line_size () const
1092 Image::stride () const
1098 Image::size () const
1104 Image::aligned () const
1111 merge (list<PositionImage> images)
1113 if (images.empty ()) {
1117 if (images.size() == 1) {
1118 return images.front ();
1121 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
1122 for (auto const& i: images) {
1123 all.extend (dcpomatic::Rect<int>(i.position, i.image->size().width, i.image->size().height));
1126 auto merged = make_shared<Image>(images.front().image->pixel_format(), dcp::Size(all.width, all.height), true);
1127 merged->make_transparent ();
1128 for (auto const& i: images) {
1129 merged->alpha_blend (i.image, i.position - all.position());
1132 return PositionImage (merged, all.position ());
1137 operator== (Image const & a, Image const & b)
1139 if (a.planes() != b.planes() || a.pixel_format() != b.pixel_format() || a.aligned() != b.aligned()) {
1143 for (int c = 0; c < a.planes(); ++c) {
1144 if (a.sample_size(c).height != b.sample_size(c).height || a.line_size()[c] != b.line_size()[c] || a.stride()[c] != b.stride()[c]) {
1148 uint8_t* p = a.data()[c];
1149 uint8_t* q = b.data()[c];
1150 int const lines = a.sample_size(c).height;
1151 for (int y = 0; y < lines; ++y) {
1152 if (memcmp (p, q, a.line_size()[c]) != 0) {
1165 * @param f Amount to fade by; 0 is black, 1 is no fade.
1168 Image::fade (float f)
1170 /* U/V black value for 8-bit colour */
1171 static int const eight_bit_uv = (1 << 7) - 1;
1172 /* U/V black value for 10-bit colour */
1173 static uint16_t const ten_bit_uv = (1 << 9) - 1;
1175 switch (_pixel_format) {
1176 case AV_PIX_FMT_YUV420P:
1179 uint8_t* p = data()[0];
1180 int const lines = sample_size(0).height;
1181 for (int y = 0; y < lines; ++y) {
1183 for (int x = 0; x < line_size()[0]; ++x) {
1184 *q = int(float(*q) * f);
1191 for (int c = 1; c < 3; ++c) {
1192 uint8_t* p = data()[c];
1193 int const lines = sample_size(c).height;
1194 for (int y = 0; y < lines; ++y) {
1196 for (int x = 0; x < line_size()[c]; ++x) {
1197 *q = eight_bit_uv + int((int(*q) - eight_bit_uv) * f);
1207 case AV_PIX_FMT_RGB24:
1210 uint8_t* p = data()[0];
1211 int const lines = sample_size(0).height;
1212 for (int y = 0; y < lines; ++y) {
1214 for (int x = 0; x < line_size()[0]; ++x) {
1215 *q = int (float (*q) * f);
1223 case AV_PIX_FMT_XYZ12LE:
1224 case AV_PIX_FMT_RGB48LE:
1225 /* 16-bit little-endian */
1226 for (int c = 0; c < 3; ++c) {
1227 int const stride_pixels = stride()[c] / 2;
1228 int const line_size_pixels = line_size()[c] / 2;
1229 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1230 int const lines = sample_size(c).height;
1231 for (int y = 0; y < lines; ++y) {
1233 for (int x = 0; x < line_size_pixels; ++x) {
1234 *q = int (float (*q) * f);
1242 case AV_PIX_FMT_YUV422P10LE:
1246 int const stride_pixels = stride()[0] / 2;
1247 int const line_size_pixels = line_size()[0] / 2;
1248 uint16_t* p = reinterpret_cast<uint16_t*> (data()[0]);
1249 int const lines = sample_size(0).height;
1250 for (int y = 0; y < lines; ++y) {
1252 for (int x = 0; x < line_size_pixels; ++x) {
1253 *q = int(float(*q) * f);
1261 for (int c = 1; c < 3; ++c) {
1262 int const stride_pixels = stride()[c] / 2;
1263 int const line_size_pixels = line_size()[c] / 2;
1264 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1265 int const lines = sample_size(c).height;
1266 for (int y = 0; y < lines; ++y) {
1268 for (int x = 0; x < line_size_pixels; ++x) {
1269 *q = ten_bit_uv + int((int(*q) - ten_bit_uv) * f);
1280 throw PixelFormatError ("fade()", _pixel_format);
1284 shared_ptr<const Image>
1285 Image::ensure_aligned (shared_ptr<const Image> image)
1287 if (image->aligned()) {
1291 return make_shared<Image>(image, true);
1295 Image::memory_used () const
1298 for (int i = 0; i < planes(); ++i) {
1299 m += _stride[i] * sample_size(i).height;
1322 png_write_data (png_structp png_ptr, png_bytep data, png_size_t length)
1324 auto mem = reinterpret_cast<Memory*>(png_get_io_ptr(png_ptr));
1325 size_t size = mem->size + length;
1328 mem->data = reinterpret_cast<uint8_t*>(realloc(mem->data, size));
1330 mem->data = reinterpret_cast<uint8_t*>(malloc(size));
1334 throw EncodeError (N_("could not allocate memory for PNG"));
1337 memcpy (mem->data + mem->size, data, length);
1338 mem->size += length;
1342 png_flush (png_structp)
1348 png_error_fn (png_structp png_ptr, char const * message)
1350 reinterpret_cast<Image*>(png_get_error_ptr(png_ptr))->png_error (message);
1354 Image::png_error (char const * message)
1356 throw EncodeError (String::compose ("Error during PNG write: %1", message));
1360 Image::as_png () const
1362 DCPOMATIC_ASSERT (bytes_per_pixel(0) == 4);
1363 DCPOMATIC_ASSERT (planes() == 1);
1364 if (pixel_format() != AV_PIX_FMT_RGBA) {
1365 return convert_pixel_format(dcp::YUVToRGB::REC709, AV_PIX_FMT_RGBA, true, false)->as_png();
1368 /* error handling? */
1369 png_structp png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, reinterpret_cast<void*>(const_cast<Image*>(this)), png_error_fn, 0);
1371 throw EncodeError (N_("could not create PNG write struct"));
1376 png_set_write_fn (png_ptr, &state, png_write_data, png_flush);
1378 png_infop info_ptr = png_create_info_struct(png_ptr);
1380 png_destroy_write_struct (&png_ptr, &info_ptr);
1381 throw EncodeError (N_("could not create PNG info struct"));
1384 png_set_IHDR (png_ptr, info_ptr, size().width, size().height, 8, PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT);
1386 png_byte ** row_pointers = reinterpret_cast<png_byte **>(png_malloc(png_ptr, size().height * sizeof(png_byte *)));
1387 for (int i = 0; i < size().height; ++i) {
1388 row_pointers[i] = (png_byte *) (data()[0] + i * stride()[0]);
1391 png_write_info (png_ptr, info_ptr);
1392 png_write_image (png_ptr, row_pointers);
1393 png_write_end (png_ptr, info_ptr);
1395 png_destroy_write_struct (&png_ptr, &info_ptr);
1396 png_free (png_ptr, row_pointers);
1398 return dcp::ArrayData (state.data, state.size);
1403 Image::video_range_to_full_range ()
1405 switch (_pixel_format) {
1406 case AV_PIX_FMT_RGB24:
1408 float const factor = 256.0 / 219.0;
1409 uint8_t* p = data()[0];
1410 int const lines = sample_size(0).height;
1411 for (int y = 0; y < lines; ++y) {
1413 for (int x = 0; x < line_size()[0]; ++x) {
1414 *q = int((*q - 16) * factor);
1421 case AV_PIX_FMT_GBRP12LE:
1423 float const factor = 4096.0 / 3504.0;
1424 for (int c = 0; c < 3; ++c) {
1425 uint16_t* p = reinterpret_cast<uint16_t*>(data()[c]);
1426 int const lines = sample_size(c).height;
1427 for (int y = 0; y < lines; ++y) {
1429 int const line_size_pixels = line_size()[c] / 2;
1430 for (int x = 0; x < line_size_pixels; ++x) {
1431 *q = int((*q - 256) * factor);
1439 throw PixelFormatError ("video_range_to_full_range()", _pixel_format);