2 Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 /** @file src/image.cc
23 * @brief A class to describe a video image.
27 #include "compose.hpp"
28 #include "dcpomatic_socket.h"
29 #include "exceptions.h"
35 #include <dcp/rgb_xyz.h>
36 #include <dcp/transfer_function.h>
37 DCPOMATIC_DISABLE_WARNINGS
39 #include <libavutil/frame.h>
40 #include <libavutil/pixdesc.h>
41 #include <libavutil/pixfmt.h>
42 #include <libswscale/swscale.h>
44 DCPOMATIC_ENABLE_WARNINGS
46 #if HAVE_VALGRIND_MEMCHECK_H
47 #include <valgrind/memcheck.h>
58 using std::make_shared;
61 using std::runtime_error;
62 using std::shared_ptr;
67 /** The memory alignment, in bytes, used for each row of an image if aligment is requested */
70 /* U/V black value for 8-bit colour */
71 static uint8_t const eight_bit_uv = (1 << 7) - 1;
72 /* U/V black value for 9-bit colour */
73 static uint16_t const nine_bit_uv = (1 << 8) - 1;
74 /* U/V black value for 10-bit colour */
75 static uint16_t const ten_bit_uv = (1 << 9) - 1;
76 /* U/V black value for 16-bit colour */
77 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
81 Image::vertical_factor (int n) const
87 auto d = av_pix_fmt_desc_get(_pixel_format);
89 throw PixelFormatError ("line_factor()", _pixel_format);
92 return lrintf(powf(2.0f, d->log2_chroma_h));
96 Image::horizontal_factor (int n) const
102 auto d = av_pix_fmt_desc_get(_pixel_format);
104 throw PixelFormatError ("sample_size()", _pixel_format);
107 return lrintf(powf(2.0f, d->log2_chroma_w));
110 /** @param n Component index.
111 * @return Number of samples (i.e. pixels, unless sub-sampled) in each direction for this component.
114 Image::sample_size (int n) const
117 lrint (ceil(static_cast<double>(size().width) / horizontal_factor(n))),
118 lrint (ceil(static_cast<double>(size().height) / vertical_factor(n)))
122 /** @return Number of planes */
124 Image::planes () const
126 auto d = av_pix_fmt_desc_get(_pixel_format);
128 throw PixelFormatError ("planes()", _pixel_format);
131 if (_pixel_format == AV_PIX_FMT_PAL8) {
135 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
139 return d->nb_components;
145 round_width_for_subsampling (int p, AVPixFmtDescriptor const * desc)
147 return p & ~ ((1 << desc->log2_chroma_w) - 1);
153 round_height_for_subsampling (int p, AVPixFmtDescriptor const * desc)
155 return p & ~ ((1 << desc->log2_chroma_h) - 1);
159 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size'.
160 * @param crop Amount to crop by.
161 * @param inter_size Size to scale the cropped image to.
162 * @param out_size Size of output frame; if this is larger than inter_size there will be black padding.
163 * @param yuv_to_rgb YUV to RGB transformation to use, if required.
164 * @param video_range Video range of the image.
165 * @param out_format Output pixel format.
166 * @param out_aligned true to make the output image aligned.
167 * @param out_video_range Video range to use for the output image.
168 * @param fast Try to be fast at the possible expense of quality; at present this means using
169 * fast bilinear rather than bicubic scaling.
172 Image::crop_scale_window (
174 dcp::Size inter_size,
176 dcp::YUVToRGB yuv_to_rgb,
177 VideoRange video_range,
178 AVPixelFormat out_format,
179 VideoRange out_video_range,
184 /* Empirical testing suggests that sws_scale() will crash if
185 the input image is not aligned.
187 DCPOMATIC_ASSERT (aligned ());
189 DCPOMATIC_ASSERT (out_size.width >= inter_size.width);
190 DCPOMATIC_ASSERT (out_size.height >= inter_size.height);
192 auto out = make_shared<Image>(out_format, out_size, out_aligned);
195 auto in_desc = av_pix_fmt_desc_get (_pixel_format);
197 throw PixelFormatError ("crop_scale_window()", _pixel_format);
200 /* Round down so that we crop only the number of pixels that is straightforward
201 * considering any subsampling.
204 round_width_for_subsampling(crop.left, in_desc),
205 round_width_for_subsampling(crop.right, in_desc),
206 round_height_for_subsampling(crop.top, in_desc),
207 round_height_for_subsampling(crop.bottom, in_desc)
210 /* Also check that we aren't cropping more image than there actually is */
211 if ((corrected_crop.left + corrected_crop.right) >= (size().width - 4)) {
212 corrected_crop.left = 0;
213 corrected_crop.right = size().width - 4;
216 if ((corrected_crop.top + corrected_crop.bottom) >= (size().height - 4)) {
217 corrected_crop.top = 0;
218 corrected_crop.bottom = size().height - 4;
221 /* Size of the image after any crop */
222 auto const cropped_size = corrected_crop.apply (size());
224 /* Scale context for a scale from cropped_size to inter_size */
225 auto scale_context = sws_getContext (
226 cropped_size.width, cropped_size.height, pixel_format(),
227 inter_size.width, inter_size.height, out_format,
228 fast ? SWS_FAST_BILINEAR : SWS_BICUBIC, 0, 0, 0
231 if (!scale_context) {
232 throw runtime_error (N_("Could not allocate SwsContext"));
235 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
236 int const lut[static_cast<int>(dcp::YUVToRGB::COUNT)] = {
241 /* The 3rd parameter here is:
242 0 -> source range MPEG (i.e. "video", 16-235)
243 1 -> source range JPEG (i.e. "full", 0-255)
245 0 -> destination range MPEG (i.e. "video", 16-235)
246 1 -> destination range JPEG (i.e. "full", 0-255)
248 But remember: sws_setColorspaceDetails ignores these
249 parameters unless the both source and destination images
250 are isYUV or isGray. (If either is not, it uses video range).
252 sws_setColorspaceDetails (
254 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), video_range == VideoRange::VIDEO ? 0 : 1,
255 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), out_video_range == VideoRange::VIDEO ? 0 : 1,
259 /* Prepare input data pointers with crop */
260 uint8_t* scale_in_data[planes()];
261 for (int c = 0; c < planes(); ++c) {
262 int const x = lrintf(bytes_per_pixel(c) * corrected_crop.left);
263 scale_in_data[c] = data()[c] + x + stride()[c] * (corrected_crop.top / vertical_factor(c));
266 auto out_desc = av_pix_fmt_desc_get (out_format);
268 throw PixelFormatError ("crop_scale_window()", out_format);
271 /* Corner of the image within out_size */
272 Position<int> const corner (
273 round_width_for_subsampling((out_size.width - inter_size.width) / 2, out_desc),
274 round_height_for_subsampling((out_size.height - inter_size.height) / 2, out_desc)
277 uint8_t* scale_out_data[out->planes()];
278 for (int c = 0; c < out->planes(); ++c) {
279 int const x = lrintf(out->bytes_per_pixel(c) * corner.x);
280 scale_out_data[c] = out->data()[c] + x + out->stride()[c] * (corner.y / out->vertical_factor(c));
285 scale_in_data, stride(),
286 0, cropped_size.height,
287 scale_out_data, out->stride()
290 sws_freeContext (scale_context);
292 if (corrected_crop != Crop() && cropped_size == inter_size) {
293 /* We are cropping without any scaling or pixel format conversion, so FFmpeg may have left some
294 data behind in our image. Clear it out. It may get to the point where we should just stop
295 trying to be clever with cropping.
297 out->make_part_black (corner.x + cropped_size.width, out_size.width - cropped_size.width);
301 video_range == VideoRange::VIDEO &&
302 out_video_range == VideoRange::FULL &&
303 av_pix_fmt_desc_get(_pixel_format)->flags & AV_PIX_FMT_FLAG_RGB
305 /* libswscale will not convert video range for RGB sources, so we have to do it ourselves */
306 out->video_range_to_full_range ();
313 Image::convert_pixel_format (dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, bool out_aligned, bool fast) const
315 return scale(size(), yuv_to_rgb, out_format, out_aligned, fast);
318 /** @param out_size Size to scale to.
319 * @param yuv_to_rgb YUVToRGB transform transform to use, if required.
320 * @param out_format Output pixel format.
321 * @param out_aligned true to make an aligned output image.
322 * @param fast Try to be fast at the possible expense of quality; at present this means using
323 * fast bilinear rather than bicubic scaling.
326 Image::scale (dcp::Size out_size, dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, bool out_aligned, bool fast) const
328 /* Empirical testing suggests that sws_scale() will crash if
329 the input image is not aligned.
331 DCPOMATIC_ASSERT (aligned ());
333 auto scaled = make_shared<Image>(out_format, out_size, out_aligned);
334 auto scale_context = sws_getContext (
335 size().width, size().height, pixel_format(),
336 out_size.width, out_size.height, out_format,
337 (fast ? SWS_FAST_BILINEAR : SWS_BICUBIC) | SWS_ACCURATE_RND, 0, 0, 0
340 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
341 int const lut[static_cast<int>(dcp::YUVToRGB::COUNT)] = {
346 /* The 3rd parameter here is:
347 0 -> source range MPEG (i.e. "video", 16-235)
348 1 -> source range JPEG (i.e. "full", 0-255)
350 0 -> destination range MPEG (i.e. "video", 16-235)
351 1 -> destination range JPEG (i.e. "full", 0-255)
353 But remember: sws_setColorspaceDetails ignores these
354 parameters unless the corresponding image isYUV or isGray.
355 (If it's neither, it uses video range).
357 sws_setColorspaceDetails (
359 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), 0,
360 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), 0,
368 scaled->data(), scaled->stride()
371 sws_freeContext (scale_context);
376 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
378 Image::yuv_16_black (uint16_t v, bool alpha)
380 memset (data()[0], 0, sample_size(0).height * stride()[0]);
381 for (int i = 1; i < 3; ++i) {
382 auto p = reinterpret_cast<int16_t*> (data()[i]);
383 int const lines = sample_size(i).height;
384 for (int y = 0; y < lines; ++y) {
385 /* We divide by 2 here because we are writing 2 bytes at a time */
386 for (int x = 0; x < line_size()[i] / 2; ++x) {
389 p += stride()[i] / 2;
394 memset (data()[3], 0, sample_size(3).height * stride()[3]);
399 Image::swap_16 (uint16_t v)
401 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
405 Image::make_part_black (int const start, int const width)
407 auto y_part = [&]() {
408 int const bpp = bytes_per_pixel(0);
409 int const h = sample_size(0).height;
410 int const s = stride()[0];
412 for (int y = 0; y < h; ++y) {
413 memset (p + start * bpp, 0, width * bpp);
418 switch (_pixel_format) {
419 case AV_PIX_FMT_RGB24:
420 case AV_PIX_FMT_ARGB:
421 case AV_PIX_FMT_RGBA:
422 case AV_PIX_FMT_ABGR:
423 case AV_PIX_FMT_BGRA:
424 case AV_PIX_FMT_RGB555LE:
425 case AV_PIX_FMT_RGB48LE:
426 case AV_PIX_FMT_RGB48BE:
427 case AV_PIX_FMT_XYZ12LE:
429 int const h = sample_size(0).height;
430 int const bpp = bytes_per_pixel(0);
431 int const s = stride()[0];
432 uint8_t* p = data()[0];
433 for (int y = 0; y < h; y++) {
434 memset (p + start * bpp, 0, width * bpp);
439 case AV_PIX_FMT_YUV420P:
442 for (int i = 1; i < 3; ++i) {
444 int const h = sample_size(i).height;
445 for (int y = 0; y < h; ++y) {
446 for (int x = start / 2; x < (start + width) / 2; ++x) {
454 case AV_PIX_FMT_YUV422P10LE:
457 for (int i = 1; i < 3; ++i) {
458 auto p = reinterpret_cast<int16_t*>(data()[i]);
459 int const h = sample_size(i).height;
460 for (int y = 0; y < h; ++y) {
461 for (int x = start / 2; x < (start + width) / 2; ++x) {
464 p += stride()[i] / 2;
470 throw PixelFormatError ("make_part_black()", _pixel_format);
477 switch (_pixel_format) {
478 case AV_PIX_FMT_YUV420P:
479 case AV_PIX_FMT_YUV422P:
480 case AV_PIX_FMT_YUV444P:
481 case AV_PIX_FMT_YUV411P:
482 memset (data()[0], 0, sample_size(0).height * stride()[0]);
483 memset (data()[1], eight_bit_uv, sample_size(1).height * stride()[1]);
484 memset (data()[2], eight_bit_uv, sample_size(2).height * stride()[2]);
487 case AV_PIX_FMT_YUVJ420P:
488 case AV_PIX_FMT_YUVJ422P:
489 case AV_PIX_FMT_YUVJ444P:
490 memset (data()[0], 0, sample_size(0).height * stride()[0]);
491 memset (data()[1], eight_bit_uv + 1, sample_size(1).height * stride()[1]);
492 memset (data()[2], eight_bit_uv + 1, sample_size(2).height * stride()[2]);
495 case AV_PIX_FMT_YUV422P9LE:
496 case AV_PIX_FMT_YUV444P9LE:
497 yuv_16_black (nine_bit_uv, false);
500 case AV_PIX_FMT_YUV422P9BE:
501 case AV_PIX_FMT_YUV444P9BE:
502 yuv_16_black (swap_16 (nine_bit_uv), false);
505 case AV_PIX_FMT_YUV422P10LE:
506 case AV_PIX_FMT_YUV444P10LE:
507 yuv_16_black (ten_bit_uv, false);
510 case AV_PIX_FMT_YUV422P16LE:
511 case AV_PIX_FMT_YUV444P16LE:
512 yuv_16_black (sixteen_bit_uv, false);
515 case AV_PIX_FMT_YUV444P10BE:
516 case AV_PIX_FMT_YUV422P10BE:
517 yuv_16_black (swap_16 (ten_bit_uv), false);
520 case AV_PIX_FMT_YUVA420P9BE:
521 case AV_PIX_FMT_YUVA422P9BE:
522 case AV_PIX_FMT_YUVA444P9BE:
523 yuv_16_black (swap_16 (nine_bit_uv), true);
526 case AV_PIX_FMT_YUVA420P9LE:
527 case AV_PIX_FMT_YUVA422P9LE:
528 case AV_PIX_FMT_YUVA444P9LE:
529 yuv_16_black (nine_bit_uv, true);
532 case AV_PIX_FMT_YUVA420P10BE:
533 case AV_PIX_FMT_YUVA422P10BE:
534 case AV_PIX_FMT_YUVA444P10BE:
535 yuv_16_black (swap_16 (ten_bit_uv), true);
538 case AV_PIX_FMT_YUVA420P10LE:
539 case AV_PIX_FMT_YUVA422P10LE:
540 case AV_PIX_FMT_YUVA444P10LE:
541 yuv_16_black (ten_bit_uv, true);
544 case AV_PIX_FMT_YUVA420P16BE:
545 case AV_PIX_FMT_YUVA422P16BE:
546 case AV_PIX_FMT_YUVA444P16BE:
547 yuv_16_black (swap_16 (sixteen_bit_uv), true);
550 case AV_PIX_FMT_YUVA420P16LE:
551 case AV_PIX_FMT_YUVA422P16LE:
552 case AV_PIX_FMT_YUVA444P16LE:
553 yuv_16_black (sixteen_bit_uv, true);
556 case AV_PIX_FMT_RGB24:
557 case AV_PIX_FMT_ARGB:
558 case AV_PIX_FMT_RGBA:
559 case AV_PIX_FMT_ABGR:
560 case AV_PIX_FMT_BGRA:
561 case AV_PIX_FMT_RGB555LE:
562 case AV_PIX_FMT_RGB48LE:
563 case AV_PIX_FMT_RGB48BE:
564 case AV_PIX_FMT_XYZ12LE:
565 memset (data()[0], 0, sample_size(0).height * stride()[0]);
568 case AV_PIX_FMT_UYVY422:
570 int const Y = sample_size(0).height;
571 int const X = line_size()[0];
572 uint8_t* p = data()[0];
573 for (int y = 0; y < Y; ++y) {
574 for (int x = 0; x < X / 4; ++x) {
575 *p++ = eight_bit_uv; // Cb
577 *p++ = eight_bit_uv; // Cr
585 throw PixelFormatError ("make_black()", _pixel_format);
590 Image::make_transparent ()
592 if (_pixel_format != AV_PIX_FMT_BGRA && _pixel_format != AV_PIX_FMT_RGBA) {
593 throw PixelFormatError ("make_transparent()", _pixel_format);
596 memset (data()[0], 0, sample_size(0).height * stride()[0]);
600 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
602 /* We're blending RGBA or BGRA images */
603 DCPOMATIC_ASSERT (other->pixel_format() == AV_PIX_FMT_BGRA || other->pixel_format() == AV_PIX_FMT_RGBA);
604 int const blue = other->pixel_format() == AV_PIX_FMT_BGRA ? 0 : 2;
605 int const red = other->pixel_format() == AV_PIX_FMT_BGRA ? 2 : 0;
607 int const other_bpp = 4;
609 int start_tx = position.x;
613 start_ox = -start_tx;
617 int start_ty = position.y;
621 start_oy = -start_ty;
625 switch (_pixel_format) {
626 case AV_PIX_FMT_RGB24:
628 /* Going onto RGB24. First byte is red, second green, third blue */
629 int const this_bpp = 3;
630 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
631 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
632 uint8_t* op = other->data()[0] + oy * other->stride()[0];
633 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
634 float const alpha = float (op[3]) / 255;
635 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
636 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
637 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
645 case AV_PIX_FMT_BGRA:
647 int const this_bpp = 4;
648 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
649 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
650 uint8_t* op = other->data()[0] + oy * other->stride()[0];
651 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
652 float const alpha = float (op[3]) / 255;
653 tp[0] = op[blue] * alpha + tp[0] * (1 - alpha);
654 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
655 tp[2] = op[red] * alpha + tp[2] * (1 - alpha);
656 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
664 case AV_PIX_FMT_RGBA:
666 int const this_bpp = 4;
667 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
668 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
669 uint8_t* op = other->data()[0] + oy * other->stride()[0];
670 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
671 float const alpha = float (op[3]) / 255;
672 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
673 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
674 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
675 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
683 case AV_PIX_FMT_RGB48LE:
685 int const this_bpp = 6;
686 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
687 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
688 uint8_t* op = other->data()[0] + oy * other->stride()[0];
689 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
690 float const alpha = float (op[3]) / 255;
691 /* Blend high bytes */
692 tp[1] = op[red] * alpha + tp[1] * (1 - alpha);
693 tp[3] = op[1] * alpha + tp[3] * (1 - alpha);
694 tp[5] = op[blue] * alpha + tp[5] * (1 - alpha);
702 case AV_PIX_FMT_XYZ12LE:
704 auto conv = dcp::ColourConversion::srgb_to_xyz();
705 double fast_matrix[9];
706 dcp::combined_rgb_to_xyz (conv, fast_matrix);
707 double const * lut_in = conv.in()->lut (8, false);
708 double const * lut_out = conv.out()->lut (16, true);
709 int const this_bpp = 6;
710 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
711 uint16_t* tp = reinterpret_cast<uint16_t*> (data()[0] + ty * stride()[0] + start_tx * this_bpp);
712 uint8_t* op = other->data()[0] + oy * other->stride()[0];
713 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
714 float const alpha = float (op[3]) / 255;
716 /* Convert sRGB to XYZ; op is BGRA. First, input gamma LUT */
717 double const r = lut_in[op[red]];
718 double const g = lut_in[op[1]];
719 double const b = lut_in[op[blue]];
721 /* RGB to XYZ, including Bradford transform and DCI companding */
722 double const x = max (0.0, min (65535.0, r * fast_matrix[0] + g * fast_matrix[1] + b * fast_matrix[2]));
723 double const y = max (0.0, min (65535.0, r * fast_matrix[3] + g * fast_matrix[4] + b * fast_matrix[5]));
724 double const z = max (0.0, min (65535.0, r * fast_matrix[6] + g * fast_matrix[7] + b * fast_matrix[8]));
726 /* Out gamma LUT and blend */
727 tp[0] = lrint(lut_out[lrint(x)] * 65535) * alpha + tp[0] * (1 - alpha);
728 tp[1] = lrint(lut_out[lrint(y)] * 65535) * alpha + tp[1] * (1 - alpha);
729 tp[2] = lrint(lut_out[lrint(z)] * 65535) * alpha + tp[2] * (1 - alpha);
737 case AV_PIX_FMT_YUV420P:
739 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, false, false);
740 dcp::Size const ts = size();
741 dcp::Size const os = yuv->size();
742 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
743 int const hty = ty / 2;
744 int const hoy = oy / 2;
745 uint8_t* tY = data()[0] + (ty * stride()[0]) + start_tx;
746 uint8_t* tU = data()[1] + (hty * stride()[1]) + start_tx / 2;
747 uint8_t* tV = data()[2] + (hty * stride()[2]) + start_tx / 2;
748 uint8_t* oY = yuv->data()[0] + (oy * yuv->stride()[0]) + start_ox;
749 uint8_t* oU = yuv->data()[1] + (hoy * yuv->stride()[1]) + start_ox / 2;
750 uint8_t* oV = yuv->data()[2] + (hoy * yuv->stride()[2]) + start_ox / 2;
751 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
752 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
753 float const a = float(alpha[3]) / 255;
754 *tY = *oY * a + *tY * (1 - a);
755 *tU = *oU * a + *tU * (1 - a);
756 *tV = *oV * a + *tV * (1 - a);
772 case AV_PIX_FMT_YUV420P10:
774 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, false, false);
775 dcp::Size const ts = size();
776 dcp::Size const os = yuv->size();
777 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
778 int const hty = ty / 2;
779 int const hoy = oy / 2;
780 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
781 uint16_t* tU = ((uint16_t *) (data()[1] + (hty * stride()[1]))) + start_tx / 2;
782 uint16_t* tV = ((uint16_t *) (data()[2] + (hty * stride()[2]))) + start_tx / 2;
783 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
784 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (hoy * yuv->stride()[1]))) + start_ox / 2;
785 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (hoy * yuv->stride()[2]))) + start_ox / 2;
786 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
787 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
788 float const a = float(alpha[3]) / 255;
789 *tY = *oY * a + *tY * (1 - a);
790 *tU = *oU * a + *tU * (1 - a);
791 *tV = *oV * a + *tV * (1 - a);
807 case AV_PIX_FMT_YUV422P10LE:
809 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, false, false);
810 dcp::Size const ts = size();
811 dcp::Size const os = yuv->size();
812 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
813 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
814 uint16_t* tU = ((uint16_t *) (data()[1] + (ty * stride()[1]))) + start_tx / 2;
815 uint16_t* tV = ((uint16_t *) (data()[2] + (ty * stride()[2]))) + start_tx / 2;
816 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
817 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (oy * yuv->stride()[1]))) + start_ox / 2;
818 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (oy * yuv->stride()[2]))) + start_ox / 2;
819 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
820 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
821 float const a = float(alpha[3]) / 255;
822 *tY = *oY * a + *tY * (1 - a);
823 *tU = *oU * a + *tU * (1 - a);
824 *tV = *oV * a + *tV * (1 - a);
841 throw PixelFormatError ("alpha_blend()", _pixel_format);
846 Image::copy (shared_ptr<const Image> other, Position<int> position)
848 /* Only implemented for RGB24 onto RGB24 so far */
849 DCPOMATIC_ASSERT (_pixel_format == AV_PIX_FMT_RGB24 && other->pixel_format() == AV_PIX_FMT_RGB24);
850 DCPOMATIC_ASSERT (position.x >= 0 && position.y >= 0);
852 int const N = min (position.x + other->size().width, size().width) - position.x;
853 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
854 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
855 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
856 memcpy (tp, op, N * 3);
861 Image::read_from_socket (shared_ptr<Socket> socket)
863 for (int i = 0; i < planes(); ++i) {
864 uint8_t* p = data()[i];
865 int const lines = sample_size(i).height;
866 for (int y = 0; y < lines; ++y) {
867 socket->read (p, line_size()[i]);
874 Image::write_to_socket (shared_ptr<Socket> socket) const
876 for (int i = 0; i < planes(); ++i) {
877 uint8_t* p = data()[i];
878 int const lines = sample_size(i).height;
879 for (int y = 0; y < lines; ++y) {
880 socket->write (p, line_size()[i]);
887 Image::bytes_per_pixel (int c) const
889 auto d = av_pix_fmt_desc_get(_pixel_format);
891 throw PixelFormatError ("bytes_per_pixel()", _pixel_format);
898 float bpp[4] = { 0, 0, 0, 0 };
900 #ifdef DCPOMATIC_HAVE_AVCOMPONENTDESCRIPTOR_DEPTH_MINUS1
901 bpp[0] = floor ((d->comp[0].depth_minus1 + 8) / 8);
902 if (d->nb_components > 1) {
903 bpp[1] = floor ((d->comp[1].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
905 if (d->nb_components > 2) {
906 bpp[2] = floor ((d->comp[2].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
908 if (d->nb_components > 3) {
909 bpp[3] = floor ((d->comp[3].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
912 bpp[0] = floor ((d->comp[0].depth + 7) / 8);
913 if (d->nb_components > 1) {
914 bpp[1] = floor ((d->comp[1].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
916 if (d->nb_components > 2) {
917 bpp[2] = floor ((d->comp[2].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
919 if (d->nb_components > 3) {
920 bpp[3] = floor ((d->comp[3].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
924 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
925 /* Not planar; sum them up */
926 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
932 /** Construct a Image of a given size and format, allocating memory
935 * @param p Pixel format.
936 * @param s Size in pixels.
937 * @param aligned true to make each row of this image aligned to a ALIGNMENT-byte boundary.
939 Image::Image (AVPixelFormat p, dcp::Size s, bool aligned)
950 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
951 _data[0] = _data[1] = _data[2] = _data[3] = 0;
953 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
954 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
956 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
957 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
959 for (int i = 0; i < planes(); ++i) {
960 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
961 _stride[i] = stride_round_up (i, _line_size, _aligned ? ALIGNMENT : 1);
963 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
964 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
965 Hence on the last pixel of the last line it reads over the end of
966 the actual data by 1 byte. If the width of an image is a multiple
967 of the stride alignment there will be no padding at the end of image lines.
968 OS X crashes on this illegal read, though other operating systems don't
969 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
970 for that instruction to read safely.
972 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
973 over-reads by more then _avx. I can't follow the code to work out how much,
974 so I'll just over-allocate by ALIGNMENT bytes and have done with it. Empirical
975 testing suggests that it works.
977 In addition to these concerns, we may read/write as much as a whole extra line
978 at the end of each plane in cases where we are messing with offsets in order to
979 do pad or crop. To solve this we over-allocate by an extra _stride[i] bytes.
981 As an example: we may write to images starting at an offset so we get some padding.
982 Hence we want to write in the following pattern:
984 block start write start line end
985 |..(padding)..|<------line-size------------->|..(padding)..|
986 |..(padding)..|<------line-size------------->|..(padding)..|
987 |..(padding)..|<------line-size------------->|..(padding)..|
989 where line-size is of the smaller (inter_size) image and the full padded line length is that of
990 out_size. To get things to work we have to tell FFmpeg that the stride is that of out_size.
991 However some parts of FFmpeg (notably rgb48Toxyz12 in swscale.c) process data for the full
992 specified *stride*. This does not matter until we get to the last line:
994 block start write start line end
995 |..(padding)..|<------line-size------------->|XXXwrittenXXX|
996 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXX|
997 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXXXXXwrittenXXX
1000 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
1001 #if HAVE_VALGRIND_MEMCHECK_H
1002 /* The data between the end of the line size and the stride is undefined but processed by
1003 libswscale, causing lots of valgrind errors. Mark it all defined to quell these errors.
1005 VALGRIND_MAKE_MEM_DEFINED (_data[i], _stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
1010 Image::Image (Image const & other)
1011 : std::enable_shared_from_this<Image>(other)
1012 , _size (other._size)
1013 , _pixel_format (other._pixel_format)
1014 , _aligned (other._aligned)
1018 for (int i = 0; i < planes(); ++i) {
1019 uint8_t* p = _data[i];
1020 uint8_t* q = other._data[i];
1021 int const lines = sample_size(i).height;
1022 for (int j = 0; j < lines; ++j) {
1023 memcpy (p, q, _line_size[i]);
1025 q += other.stride()[i];
1030 Image::Image (AVFrame const * frame)
1031 : _size (frame->width, frame->height)
1032 , _pixel_format (static_cast<AVPixelFormat>(frame->format))
1035 DCPOMATIC_ASSERT (_pixel_format != AV_PIX_FMT_NONE);
1039 for (int i = 0; i < planes(); ++i) {
1040 uint8_t* p = _data[i];
1041 uint8_t* q = frame->data[i];
1042 int const lines = sample_size(i).height;
1043 for (int j = 0; j < lines; ++j) {
1044 memcpy (p, q, _line_size[i]);
1046 /* AVFrame's linesize is what we call `stride' */
1047 q += frame->linesize[i];
1052 Image::Image (shared_ptr<const Image> other, bool aligned)
1053 : _size (other->_size)
1054 , _pixel_format (other->_pixel_format)
1055 , _aligned (aligned)
1059 for (int i = 0; i < planes(); ++i) {
1060 DCPOMATIC_ASSERT (line_size()[i] == other->line_size()[i]);
1061 uint8_t* p = _data[i];
1062 uint8_t* q = other->data()[i];
1063 int const lines = sample_size(i).height;
1064 for (int j = 0; j < lines; ++j) {
1065 memcpy (p, q, line_size()[i]);
1067 q += other->stride()[i];
1073 Image::operator= (Image const & other)
1075 if (this == &other) {
1085 Image::swap (Image & other)
1087 std::swap (_size, other._size);
1088 std::swap (_pixel_format, other._pixel_format);
1090 for (int i = 0; i < 4; ++i) {
1091 std::swap (_data[i], other._data[i]);
1092 std::swap (_line_size[i], other._line_size[i]);
1093 std::swap (_stride[i], other._stride[i]);
1096 std::swap (_aligned, other._aligned);
1101 for (int i = 0; i < planes(); ++i) {
1106 av_free (_line_size);
1111 Image::data () const
1117 Image::line_size () const
1123 Image::stride () const
1129 Image::size () const
1135 Image::aligned () const
1142 merge (list<PositionImage> images)
1144 if (images.empty ()) {
1148 if (images.size() == 1) {
1149 return images.front ();
1152 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
1153 for (auto const& i: images) {
1154 all.extend (dcpomatic::Rect<int>(i.position, i.image->size().width, i.image->size().height));
1157 auto merged = make_shared<Image>(images.front().image->pixel_format(), dcp::Size(all.width, all.height), true);
1158 merged->make_transparent ();
1159 for (auto const& i: images) {
1160 merged->alpha_blend (i.image, i.position - all.position());
1163 return PositionImage (merged, all.position ());
1168 operator== (Image const & a, Image const & b)
1170 if (a.planes() != b.planes() || a.pixel_format() != b.pixel_format() || a.aligned() != b.aligned()) {
1174 for (int c = 0; c < a.planes(); ++c) {
1175 if (a.sample_size(c).height != b.sample_size(c).height || a.line_size()[c] != b.line_size()[c] || a.stride()[c] != b.stride()[c]) {
1179 uint8_t* p = a.data()[c];
1180 uint8_t* q = b.data()[c];
1181 int const lines = a.sample_size(c).height;
1182 for (int y = 0; y < lines; ++y) {
1183 if (memcmp (p, q, a.line_size()[c]) != 0) {
1196 * @param f Amount to fade by; 0 is black, 1 is no fade.
1199 Image::fade (float f)
1201 /* U/V black value for 8-bit colour */
1202 static int const eight_bit_uv = (1 << 7) - 1;
1203 /* U/V black value for 10-bit colour */
1204 static uint16_t const ten_bit_uv = (1 << 9) - 1;
1206 switch (_pixel_format) {
1207 case AV_PIX_FMT_YUV420P:
1210 uint8_t* p = data()[0];
1211 int const lines = sample_size(0).height;
1212 for (int y = 0; y < lines; ++y) {
1214 for (int x = 0; x < line_size()[0]; ++x) {
1215 *q = int(float(*q) * f);
1222 for (int c = 1; c < 3; ++c) {
1223 uint8_t* p = data()[c];
1224 int const lines = sample_size(c).height;
1225 for (int y = 0; y < lines; ++y) {
1227 for (int x = 0; x < line_size()[c]; ++x) {
1228 *q = eight_bit_uv + int((int(*q) - eight_bit_uv) * f);
1238 case AV_PIX_FMT_RGB24:
1241 uint8_t* p = data()[0];
1242 int const lines = sample_size(0).height;
1243 for (int y = 0; y < lines; ++y) {
1245 for (int x = 0; x < line_size()[0]; ++x) {
1246 *q = int (float (*q) * f);
1254 case AV_PIX_FMT_XYZ12LE:
1255 case AV_PIX_FMT_RGB48LE:
1256 /* 16-bit little-endian */
1257 for (int c = 0; c < 3; ++c) {
1258 int const stride_pixels = stride()[c] / 2;
1259 int const line_size_pixels = line_size()[c] / 2;
1260 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1261 int const lines = sample_size(c).height;
1262 for (int y = 0; y < lines; ++y) {
1264 for (int x = 0; x < line_size_pixels; ++x) {
1265 *q = int (float (*q) * f);
1273 case AV_PIX_FMT_YUV422P10LE:
1277 int const stride_pixels = stride()[0] / 2;
1278 int const line_size_pixels = line_size()[0] / 2;
1279 uint16_t* p = reinterpret_cast<uint16_t*> (data()[0]);
1280 int const lines = sample_size(0).height;
1281 for (int y = 0; y < lines; ++y) {
1283 for (int x = 0; x < line_size_pixels; ++x) {
1284 *q = int(float(*q) * f);
1292 for (int c = 1; c < 3; ++c) {
1293 int const stride_pixels = stride()[c] / 2;
1294 int const line_size_pixels = line_size()[c] / 2;
1295 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1296 int const lines = sample_size(c).height;
1297 for (int y = 0; y < lines; ++y) {
1299 for (int x = 0; x < line_size_pixels; ++x) {
1300 *q = ten_bit_uv + int((int(*q) - ten_bit_uv) * f);
1311 throw PixelFormatError ("fade()", _pixel_format);
1315 shared_ptr<const Image>
1316 Image::ensure_aligned (shared_ptr<const Image> image)
1318 if (image->aligned()) {
1322 return make_shared<Image>(image, true);
1326 Image::memory_used () const
1329 for (int i = 0; i < planes(); ++i) {
1330 m += _stride[i] * sample_size(i).height;
1353 png_write_data (png_structp png_ptr, png_bytep data, png_size_t length)
1355 auto mem = reinterpret_cast<Memory*>(png_get_io_ptr(png_ptr));
1356 size_t size = mem->size + length;
1359 mem->data = reinterpret_cast<uint8_t*>(realloc(mem->data, size));
1361 mem->data = reinterpret_cast<uint8_t*>(malloc(size));
1365 throw EncodeError (N_("could not allocate memory for PNG"));
1368 memcpy (mem->data + mem->size, data, length);
1369 mem->size += length;
1373 png_flush (png_structp)
1379 png_error_fn (png_structp png_ptr, char const * message)
1381 reinterpret_cast<Image*>(png_get_error_ptr(png_ptr))->png_error (message);
1385 Image::png_error (char const * message)
1387 throw EncodeError (String::compose ("Error during PNG write: %1", message));
1391 Image::as_png () const
1393 DCPOMATIC_ASSERT (bytes_per_pixel(0) == 4);
1394 DCPOMATIC_ASSERT (planes() == 1);
1395 if (pixel_format() != AV_PIX_FMT_RGBA) {
1396 return convert_pixel_format(dcp::YUVToRGB::REC709, AV_PIX_FMT_RGBA, true, false)->as_png();
1399 /* error handling? */
1400 png_structp png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, reinterpret_cast<void*>(const_cast<Image*>(this)), png_error_fn, 0);
1402 throw EncodeError (N_("could not create PNG write struct"));
1407 png_set_write_fn (png_ptr, &state, png_write_data, png_flush);
1409 png_infop info_ptr = png_create_info_struct(png_ptr);
1411 png_destroy_write_struct (&png_ptr, &info_ptr);
1412 throw EncodeError (N_("could not create PNG info struct"));
1415 png_set_IHDR (png_ptr, info_ptr, size().width, size().height, 8, PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT);
1417 png_byte ** row_pointers = reinterpret_cast<png_byte **>(png_malloc(png_ptr, size().height * sizeof(png_byte *)));
1418 for (int i = 0; i < size().height; ++i) {
1419 row_pointers[i] = (png_byte *) (data()[0] + i * stride()[0]);
1422 png_write_info (png_ptr, info_ptr);
1423 png_write_image (png_ptr, row_pointers);
1424 png_write_end (png_ptr, info_ptr);
1426 png_destroy_write_struct (&png_ptr, &info_ptr);
1427 png_free (png_ptr, row_pointers);
1429 return dcp::ArrayData (state.data, state.size);
1434 Image::video_range_to_full_range ()
1436 switch (_pixel_format) {
1437 case AV_PIX_FMT_RGB24:
1439 float const factor = 256.0 / 219.0;
1440 uint8_t* p = data()[0];
1441 int const lines = sample_size(0).height;
1442 for (int y = 0; y < lines; ++y) {
1444 for (int x = 0; x < line_size()[0]; ++x) {
1445 *q = clamp(lrintf((*q - 16) * factor), 0L, 255L);
1452 case AV_PIX_FMT_RGB48LE:
1454 float const factor = 65536.0 / 56064.0;
1455 uint16_t* p = reinterpret_cast<uint16_t*>(data()[0]);
1456 int const lines = sample_size(0).height;
1457 for (int y = 0; y < lines; ++y) {
1459 int const line_size_pixels = line_size()[0] / 2;
1460 for (int x = 0; x < line_size_pixels; ++x) {
1461 *q = clamp(lrintf((*q - 4096) * factor), 0L, 65535L);
1464 p += stride()[0] / 2;
1468 case AV_PIX_FMT_GBRP12LE:
1470 float const factor = 4096.0 / 3504.0;
1471 for (int c = 0; c < 3; ++c) {
1472 uint16_t* p = reinterpret_cast<uint16_t*>(data()[c]);
1473 int const lines = sample_size(c).height;
1474 for (int y = 0; y < lines; ++y) {
1476 int const line_size_pixels = line_size()[c] / 2;
1477 for (int x = 0; x < line_size_pixels; ++x) {
1478 *q = clamp(lrintf((*q - 256) * factor), 0L, 4095L);
1486 throw PixelFormatError ("video_range_to_full_range()", _pixel_format);