2 Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 /** @file src/image.cc
23 * @brief A class to describe a video image.
27 #include "compose.hpp"
28 #include "dcpomatic_assert.h"
29 #include "dcpomatic_socket.h"
30 #include "exceptions.h"
32 #include "maths_util.h"
37 #include <dcp/rgb_xyz.h>
38 #include <dcp/transfer_function.h>
39 DCPOMATIC_DISABLE_WARNINGS
41 #include <libavutil/frame.h>
42 #include <libavutil/pixdesc.h>
43 #include <libavutil/pixfmt.h>
44 #include <libswscale/swscale.h>
46 DCPOMATIC_ENABLE_WARNINGS
47 #if HAVE_VALGRIND_MEMCHECK_H
48 #include <valgrind/memcheck.h>
59 using std::make_shared;
62 using std::runtime_error;
63 using std::shared_ptr;
68 /** The memory alignment, in bytes, used for each row of an image if Alignment::PADDED is requested */
69 int constexpr ALIGNMENT = 64;
71 /* U/V black value for 8-bit colour */
72 static uint8_t const eight_bit_uv = (1 << 7) - 1;
73 /* U/V black value for 9-bit colour */
74 static uint16_t const nine_bit_uv = (1 << 8) - 1;
75 /* U/V black value for 10-bit colour */
76 static uint16_t const ten_bit_uv = (1 << 9) - 1;
77 /* U/V black value for 16-bit colour */
78 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
82 Image::vertical_factor (int n) const
88 auto d = av_pix_fmt_desc_get(_pixel_format);
90 throw PixelFormatError ("line_factor()", _pixel_format);
93 return lrintf(powf(2.0f, d->log2_chroma_h));
97 Image::horizontal_factor (int n) const
103 auto d = av_pix_fmt_desc_get(_pixel_format);
105 throw PixelFormatError ("sample_size()", _pixel_format);
108 return lrintf(powf(2.0f, d->log2_chroma_w));
112 /** @param n Component index.
113 * @return Number of samples (i.e. pixels, unless sub-sampled) in each direction for this component.
116 Image::sample_size (int n) const
119 lrint (ceil(static_cast<double>(size().width) / horizontal_factor(n))),
120 lrint (ceil(static_cast<double>(size().height) / vertical_factor(n)))
125 /** @return Number of planes */
127 Image::planes () const
129 if (_pixel_format == AV_PIX_FMT_PAL8) {
133 auto d = av_pix_fmt_desc_get(_pixel_format);
135 throw PixelFormatError ("planes()", _pixel_format);
138 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
142 return d->nb_components;
148 round_width_for_subsampling (int p, AVPixFmtDescriptor const * desc)
150 return p & ~ ((1 << desc->log2_chroma_w) - 1);
156 round_height_for_subsampling (int p, AVPixFmtDescriptor const * desc)
158 return p & ~ ((1 << desc->log2_chroma_h) - 1);
162 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size'.
163 * @param crop Amount to crop by.
164 * @param inter_size Size to scale the cropped image to.
165 * @param out_size Size of output frame; if this is larger than inter_size there will be black padding.
166 * @param yuv_to_rgb YUV to RGB transformation to use, if required.
167 * @param video_range Video range of the image.
168 * @param out_format Output pixel format.
169 * @param out_aligned true to make the output image aligned.
170 * @param out_video_range Video range to use for the output image.
171 * @param fast Try to be fast at the possible expense of quality; at present this means using
172 * fast bilinear rather than bicubic scaling.
175 Image::crop_scale_window (
177 dcp::Size inter_size,
179 dcp::YUVToRGB yuv_to_rgb,
180 VideoRange video_range,
181 AVPixelFormat out_format,
182 VideoRange out_video_range,
183 Alignment out_alignment,
187 /* Empirical testing suggests that sws_scale() will crash if
188 the input image is not padded.
190 DCPOMATIC_ASSERT (alignment() == Alignment::PADDED);
192 DCPOMATIC_ASSERT (out_size.width >= inter_size.width);
193 DCPOMATIC_ASSERT (out_size.height >= inter_size.height);
195 auto out = make_shared<Image>(out_format, out_size, out_alignment);
198 auto in_desc = av_pix_fmt_desc_get (_pixel_format);
200 throw PixelFormatError ("crop_scale_window()", _pixel_format);
203 /* Round down so that we crop only the number of pixels that is straightforward
204 * considering any subsampling.
207 round_width_for_subsampling(crop.left, in_desc),
208 round_width_for_subsampling(crop.right, in_desc),
209 round_height_for_subsampling(crop.top, in_desc),
210 round_height_for_subsampling(crop.bottom, in_desc)
213 /* Also check that we aren't cropping more image than there actually is */
214 if ((corrected_crop.left + corrected_crop.right) >= (size().width - 4)) {
215 corrected_crop.left = 0;
216 corrected_crop.right = size().width - 4;
219 if ((corrected_crop.top + corrected_crop.bottom) >= (size().height - 4)) {
220 corrected_crop.top = 0;
221 corrected_crop.bottom = size().height - 4;
224 /* Size of the image after any crop */
225 auto const cropped_size = corrected_crop.apply (size());
227 /* Scale context for a scale from cropped_size to inter_size */
228 auto scale_context = sws_getContext (
229 cropped_size.width, cropped_size.height, pixel_format(),
230 inter_size.width, inter_size.height, out_format,
231 fast ? SWS_FAST_BILINEAR : SWS_BICUBIC, 0, 0, 0
234 if (!scale_context) {
235 throw runtime_error (N_("Could not allocate SwsContext"));
238 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
239 int const lut[static_cast<int>(dcp::YUVToRGB::COUNT)] = {
244 /* The 3rd parameter here is:
245 0 -> source range MPEG (i.e. "video", 16-235)
246 1 -> source range JPEG (i.e. "full", 0-255)
248 0 -> destination range MPEG (i.e. "video", 16-235)
249 1 -> destination range JPEG (i.e. "full", 0-255)
251 But remember: sws_setColorspaceDetails ignores these
252 parameters unless the both source and destination images
253 are isYUV or isGray. (If either is not, it uses video range).
255 sws_setColorspaceDetails (
257 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), video_range == VideoRange::VIDEO ? 0 : 1,
258 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), out_video_range == VideoRange::VIDEO ? 0 : 1,
262 /* Prepare input data pointers with crop */
263 uint8_t* scale_in_data[planes()];
264 for (int c = 0; c < planes(); ++c) {
265 int const x = lrintf(bytes_per_pixel(c) * corrected_crop.left);
266 scale_in_data[c] = data()[c] + x + stride()[c] * (corrected_crop.top / vertical_factor(c));
269 auto out_desc = av_pix_fmt_desc_get (out_format);
271 throw PixelFormatError ("crop_scale_window()", out_format);
274 /* Corner of the image within out_size */
275 Position<int> const corner (
276 round_width_for_subsampling((out_size.width - inter_size.width) / 2, out_desc),
277 round_height_for_subsampling((out_size.height - inter_size.height) / 2, out_desc)
280 uint8_t* scale_out_data[out->planes()];
281 for (int c = 0; c < out->planes(); ++c) {
282 int const x = lrintf(out->bytes_per_pixel(c) * corner.x);
283 scale_out_data[c] = out->data()[c] + x + out->stride()[c] * (corner.y / out->vertical_factor(c));
288 scale_in_data, stride(),
289 0, cropped_size.height,
290 scale_out_data, out->stride()
293 sws_freeContext (scale_context);
295 if (corrected_crop != Crop() && cropped_size == inter_size) {
296 /* We are cropping without any scaling or pixel format conversion, so FFmpeg may have left some
297 data behind in our image. Clear it out. It may get to the point where we should just stop
298 trying to be clever with cropping.
300 out->make_part_black (corner.x + cropped_size.width, out_size.width - cropped_size.width);
304 video_range == VideoRange::VIDEO &&
305 out_video_range == VideoRange::FULL &&
306 av_pix_fmt_desc_get(_pixel_format)->flags & AV_PIX_FMT_FLAG_RGB
308 /* libswscale will not convert video range for RGB sources, so we have to do it ourselves */
309 out->video_range_to_full_range ();
317 Image::convert_pixel_format (dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, Alignment out_alignment, bool fast) const
319 return scale(size(), yuv_to_rgb, out_format, out_alignment, fast);
323 /** @param out_size Size to scale to.
324 * @param yuv_to_rgb YUVToRGB transform transform to use, if required.
325 * @param out_format Output pixel format.
326 * @param out_aligment Output alignment.
327 * @param fast Try to be fast at the possible expense of quality; at present this means using
328 * fast bilinear rather than bicubic scaling.
331 Image::scale (dcp::Size out_size, dcp::YUVToRGB yuv_to_rgb, AVPixelFormat out_format, Alignment out_alignment, bool fast) const
333 /* Empirical testing suggests that sws_scale() will crash if
334 the input image alignment is not PADDED.
336 DCPOMATIC_ASSERT (alignment() == Alignment::PADDED);
338 auto scaled = make_shared<Image>(out_format, out_size, out_alignment);
339 auto scale_context = sws_getContext (
340 size().width, size().height, pixel_format(),
341 out_size.width, out_size.height, out_format,
342 (fast ? SWS_FAST_BILINEAR : SWS_BICUBIC) | SWS_ACCURATE_RND, 0, 0, 0
345 DCPOMATIC_ASSERT (yuv_to_rgb < dcp::YUVToRGB::COUNT);
346 int const lut[static_cast<int>(dcp::YUVToRGB::COUNT)] = {
351 /* The 3rd parameter here is:
352 0 -> source range MPEG (i.e. "video", 16-235)
353 1 -> source range JPEG (i.e. "full", 0-255)
355 0 -> destination range MPEG (i.e. "video", 16-235)
356 1 -> destination range JPEG (i.e. "full", 0-255)
358 But remember: sws_setColorspaceDetails ignores these
359 parameters unless the corresponding image isYUV or isGray.
360 (If it's neither, it uses video range).
362 sws_setColorspaceDetails (
364 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), 0,
365 sws_getCoefficients (lut[static_cast<int>(yuv_to_rgb)]), 0,
373 scaled->data(), scaled->stride()
376 sws_freeContext (scale_context);
382 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
384 Image::yuv_16_black (uint16_t v, bool alpha)
386 memset (data()[0], 0, sample_size(0).height * stride()[0]);
387 for (int i = 1; i < 3; ++i) {
388 auto p = reinterpret_cast<int16_t*> (data()[i]);
389 int const lines = sample_size(i).height;
390 for (int y = 0; y < lines; ++y) {
391 /* We divide by 2 here because we are writing 2 bytes at a time */
392 for (int x = 0; x < line_size()[i] / 2; ++x) {
395 p += stride()[i] / 2;
400 memset (data()[3], 0, sample_size(3).height * stride()[3]);
406 Image::swap_16 (uint16_t v)
408 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
413 Image::make_part_black (int const start, int const width)
415 auto y_part = [&]() {
416 int const bpp = bytes_per_pixel(0);
417 int const h = sample_size(0).height;
418 int const s = stride()[0];
420 for (int y = 0; y < h; ++y) {
421 memset (p + start * bpp, 0, width * bpp);
426 switch (_pixel_format) {
427 case AV_PIX_FMT_RGB24:
428 case AV_PIX_FMT_ARGB:
429 case AV_PIX_FMT_RGBA:
430 case AV_PIX_FMT_ABGR:
431 case AV_PIX_FMT_BGRA:
432 case AV_PIX_FMT_RGB555LE:
433 case AV_PIX_FMT_RGB48LE:
434 case AV_PIX_FMT_RGB48BE:
435 case AV_PIX_FMT_XYZ12LE:
437 int const h = sample_size(0).height;
438 int const bpp = bytes_per_pixel(0);
439 int const s = stride()[0];
440 uint8_t* p = data()[0];
441 for (int y = 0; y < h; y++) {
442 memset (p + start * bpp, 0, width * bpp);
447 case AV_PIX_FMT_YUV420P:
450 for (int i = 1; i < 3; ++i) {
452 int const h = sample_size(i).height;
453 for (int y = 0; y < h; ++y) {
454 for (int x = start / 2; x < (start + width) / 2; ++x) {
462 case AV_PIX_FMT_YUV422P10LE:
465 for (int i = 1; i < 3; ++i) {
466 auto p = reinterpret_cast<int16_t*>(data()[i]);
467 int const h = sample_size(i).height;
468 for (int y = 0; y < h; ++y) {
469 for (int x = start / 2; x < (start + width) / 2; ++x) {
472 p += stride()[i] / 2;
478 throw PixelFormatError ("make_part_black()", _pixel_format);
486 switch (_pixel_format) {
487 case AV_PIX_FMT_YUV420P:
488 case AV_PIX_FMT_YUV422P:
489 case AV_PIX_FMT_YUV444P:
490 case AV_PIX_FMT_YUV411P:
491 memset (data()[0], 0, sample_size(0).height * stride()[0]);
492 memset (data()[1], eight_bit_uv, sample_size(1).height * stride()[1]);
493 memset (data()[2], eight_bit_uv, sample_size(2).height * stride()[2]);
496 case AV_PIX_FMT_YUVJ420P:
497 case AV_PIX_FMT_YUVJ422P:
498 case AV_PIX_FMT_YUVJ444P:
499 memset (data()[0], 0, sample_size(0).height * stride()[0]);
500 memset (data()[1], eight_bit_uv + 1, sample_size(1).height * stride()[1]);
501 memset (data()[2], eight_bit_uv + 1, sample_size(2).height * stride()[2]);
504 case AV_PIX_FMT_YUV422P9LE:
505 case AV_PIX_FMT_YUV444P9LE:
506 yuv_16_black (nine_bit_uv, false);
509 case AV_PIX_FMT_YUV422P9BE:
510 case AV_PIX_FMT_YUV444P9BE:
511 yuv_16_black (swap_16 (nine_bit_uv), false);
514 case AV_PIX_FMT_YUV422P10LE:
515 case AV_PIX_FMT_YUV444P10LE:
516 yuv_16_black (ten_bit_uv, false);
519 case AV_PIX_FMT_YUV422P16LE:
520 case AV_PIX_FMT_YUV444P16LE:
521 yuv_16_black (sixteen_bit_uv, false);
524 case AV_PIX_FMT_YUV444P10BE:
525 case AV_PIX_FMT_YUV422P10BE:
526 yuv_16_black (swap_16 (ten_bit_uv), false);
529 case AV_PIX_FMT_YUVA420P9BE:
530 case AV_PIX_FMT_YUVA422P9BE:
531 case AV_PIX_FMT_YUVA444P9BE:
532 yuv_16_black (swap_16 (nine_bit_uv), true);
535 case AV_PIX_FMT_YUVA420P9LE:
536 case AV_PIX_FMT_YUVA422P9LE:
537 case AV_PIX_FMT_YUVA444P9LE:
538 yuv_16_black (nine_bit_uv, true);
541 case AV_PIX_FMT_YUVA420P10BE:
542 case AV_PIX_FMT_YUVA422P10BE:
543 case AV_PIX_FMT_YUVA444P10BE:
544 yuv_16_black (swap_16 (ten_bit_uv), true);
547 case AV_PIX_FMT_YUVA420P10LE:
548 case AV_PIX_FMT_YUVA422P10LE:
549 case AV_PIX_FMT_YUVA444P10LE:
550 yuv_16_black (ten_bit_uv, true);
553 case AV_PIX_FMT_YUVA420P16BE:
554 case AV_PIX_FMT_YUVA422P16BE:
555 case AV_PIX_FMT_YUVA444P16BE:
556 yuv_16_black (swap_16 (sixteen_bit_uv), true);
559 case AV_PIX_FMT_YUVA420P16LE:
560 case AV_PIX_FMT_YUVA422P16LE:
561 case AV_PIX_FMT_YUVA444P16LE:
562 yuv_16_black (sixteen_bit_uv, true);
565 case AV_PIX_FMT_RGB24:
566 case AV_PIX_FMT_ARGB:
567 case AV_PIX_FMT_RGBA:
568 case AV_PIX_FMT_ABGR:
569 case AV_PIX_FMT_BGRA:
570 case AV_PIX_FMT_RGB555LE:
571 case AV_PIX_FMT_RGB48LE:
572 case AV_PIX_FMT_RGB48BE:
573 case AV_PIX_FMT_XYZ12LE:
574 memset (data()[0], 0, sample_size(0).height * stride()[0]);
577 case AV_PIX_FMT_UYVY422:
579 int const Y = sample_size(0).height;
580 int const X = line_size()[0];
581 uint8_t* p = data()[0];
582 for (int y = 0; y < Y; ++y) {
583 for (int x = 0; x < X / 4; ++x) {
584 *p++ = eight_bit_uv; // Cb
586 *p++ = eight_bit_uv; // Cr
594 throw PixelFormatError ("make_black()", _pixel_format);
600 Image::make_transparent ()
602 if (_pixel_format != AV_PIX_FMT_BGRA && _pixel_format != AV_PIX_FMT_RGBA) {
603 throw PixelFormatError ("make_transparent()", _pixel_format);
606 memset (data()[0], 0, sample_size(0).height * stride()[0]);
611 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
613 /* We're blending RGBA or BGRA images */
614 DCPOMATIC_ASSERT (other->pixel_format() == AV_PIX_FMT_BGRA || other->pixel_format() == AV_PIX_FMT_RGBA);
615 int const blue = other->pixel_format() == AV_PIX_FMT_BGRA ? 0 : 2;
616 int const red = other->pixel_format() == AV_PIX_FMT_BGRA ? 2 : 0;
618 int const other_bpp = 4;
620 int start_tx = position.x;
624 start_ox = -start_tx;
628 int start_ty = position.y;
632 start_oy = -start_ty;
636 switch (_pixel_format) {
637 case AV_PIX_FMT_RGB24:
639 /* Going onto RGB24. First byte is red, second green, third blue */
640 int const this_bpp = 3;
641 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
642 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
643 uint8_t* op = other->data()[0] + oy * other->stride()[0];
644 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
645 float const alpha = float (op[3]) / 255;
646 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
647 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
648 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
656 case AV_PIX_FMT_BGRA:
658 int const this_bpp = 4;
659 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
660 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
661 uint8_t* op = other->data()[0] + oy * other->stride()[0];
662 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
663 float const alpha = float (op[3]) / 255;
664 tp[0] = op[blue] * alpha + tp[0] * (1 - alpha);
665 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
666 tp[2] = op[red] * alpha + tp[2] * (1 - alpha);
667 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
675 case AV_PIX_FMT_RGBA:
677 int const this_bpp = 4;
678 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
679 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
680 uint8_t* op = other->data()[0] + oy * other->stride()[0];
681 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
682 float const alpha = float (op[3]) / 255;
683 tp[0] = op[red] * alpha + tp[0] * (1 - alpha);
684 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
685 tp[2] = op[blue] * alpha + tp[2] * (1 - alpha);
686 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
694 case AV_PIX_FMT_RGB48LE:
696 int const this_bpp = 6;
697 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
698 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
699 uint8_t* op = other->data()[0] + oy * other->stride()[0];
700 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
701 float const alpha = float (op[3]) / 255;
702 /* Blend high bytes */
703 tp[1] = op[red] * alpha + tp[1] * (1 - alpha);
704 tp[3] = op[1] * alpha + tp[3] * (1 - alpha);
705 tp[5] = op[blue] * alpha + tp[5] * (1 - alpha);
713 case AV_PIX_FMT_XYZ12LE:
715 auto conv = dcp::ColourConversion::srgb_to_xyz();
716 double fast_matrix[9];
717 dcp::combined_rgb_to_xyz (conv, fast_matrix);
718 double const * lut_in = conv.in()->lut (8, false);
719 double const * lut_out = conv.out()->lut (16, true);
720 int const this_bpp = 6;
721 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
722 uint16_t* tp = reinterpret_cast<uint16_t*> (data()[0] + ty * stride()[0] + start_tx * this_bpp);
723 uint8_t* op = other->data()[0] + oy * other->stride()[0];
724 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
725 float const alpha = float (op[3]) / 255;
727 /* Convert sRGB to XYZ; op is BGRA. First, input gamma LUT */
728 double const r = lut_in[op[red]];
729 double const g = lut_in[op[1]];
730 double const b = lut_in[op[blue]];
732 /* RGB to XYZ, including Bradford transform and DCI companding */
733 double const x = max (0.0, min (65535.0, r * fast_matrix[0] + g * fast_matrix[1] + b * fast_matrix[2]));
734 double const y = max (0.0, min (65535.0, r * fast_matrix[3] + g * fast_matrix[4] + b * fast_matrix[5]));
735 double const z = max (0.0, min (65535.0, r * fast_matrix[6] + g * fast_matrix[7] + b * fast_matrix[8]));
737 /* Out gamma LUT and blend */
738 tp[0] = lrint(lut_out[lrint(x)] * 65535) * alpha + tp[0] * (1 - alpha);
739 tp[1] = lrint(lut_out[lrint(y)] * 65535) * alpha + tp[1] * (1 - alpha);
740 tp[2] = lrint(lut_out[lrint(z)] * 65535) * alpha + tp[2] * (1 - alpha);
748 case AV_PIX_FMT_YUV420P:
750 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
751 dcp::Size const ts = size();
752 dcp::Size const os = yuv->size();
753 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
754 int const hty = ty / 2;
755 int const hoy = oy / 2;
756 uint8_t* tY = data()[0] + (ty * stride()[0]) + start_tx;
757 uint8_t* tU = data()[1] + (hty * stride()[1]) + start_tx / 2;
758 uint8_t* tV = data()[2] + (hty * stride()[2]) + start_tx / 2;
759 uint8_t* oY = yuv->data()[0] + (oy * yuv->stride()[0]) + start_ox;
760 uint8_t* oU = yuv->data()[1] + (hoy * yuv->stride()[1]) + start_ox / 2;
761 uint8_t* oV = yuv->data()[2] + (hoy * yuv->stride()[2]) + start_ox / 2;
762 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
763 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
764 float const a = float(alpha[3]) / 255;
765 *tY = *oY * a + *tY * (1 - a);
766 *tU = *oU * a + *tU * (1 - a);
767 *tV = *oV * a + *tV * (1 - a);
783 case AV_PIX_FMT_YUV420P10:
785 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
786 dcp::Size const ts = size();
787 dcp::Size const os = yuv->size();
788 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
789 int const hty = ty / 2;
790 int const hoy = oy / 2;
791 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
792 uint16_t* tU = ((uint16_t *) (data()[1] + (hty * stride()[1]))) + start_tx / 2;
793 uint16_t* tV = ((uint16_t *) (data()[2] + (hty * stride()[2]))) + start_tx / 2;
794 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
795 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (hoy * yuv->stride()[1]))) + start_ox / 2;
796 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (hoy * yuv->stride()[2]))) + start_ox / 2;
797 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
798 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
799 float const a = float(alpha[3]) / 255;
800 *tY = *oY * a + *tY * (1 - a);
801 *tU = *oU * a + *tU * (1 - a);
802 *tV = *oV * a + *tV * (1 - a);
818 case AV_PIX_FMT_YUV422P10LE:
820 auto yuv = other->convert_pixel_format (dcp::YUVToRGB::REC709, _pixel_format, Alignment::COMPACT, false);
821 dcp::Size const ts = size();
822 dcp::Size const os = yuv->size();
823 for (int ty = start_ty, oy = start_oy; ty < ts.height && oy < os.height; ++ty, ++oy) {
824 uint16_t* tY = ((uint16_t *) (data()[0] + (ty * stride()[0]))) + start_tx;
825 uint16_t* tU = ((uint16_t *) (data()[1] + (ty * stride()[1]))) + start_tx / 2;
826 uint16_t* tV = ((uint16_t *) (data()[2] + (ty * stride()[2]))) + start_tx / 2;
827 uint16_t* oY = ((uint16_t *) (yuv->data()[0] + (oy * yuv->stride()[0]))) + start_ox;
828 uint16_t* oU = ((uint16_t *) (yuv->data()[1] + (oy * yuv->stride()[1]))) + start_ox / 2;
829 uint16_t* oV = ((uint16_t *) (yuv->data()[2] + (oy * yuv->stride()[2]))) + start_ox / 2;
830 uint8_t* alpha = other->data()[0] + (oy * other->stride()[0]) + start_ox * 4;
831 for (int tx = start_tx, ox = start_ox; tx < ts.width && ox < os.width; ++tx, ++ox) {
832 float const a = float(alpha[3]) / 255;
833 *tY = *oY * a + *tY * (1 - a);
834 *tU = *oU * a + *tU * (1 - a);
835 *tV = *oV * a + *tV * (1 - a);
852 throw PixelFormatError ("alpha_blend()", _pixel_format);
858 Image::copy (shared_ptr<const Image> other, Position<int> position)
860 /* Only implemented for RGB24 onto RGB24 so far */
861 DCPOMATIC_ASSERT (_pixel_format == AV_PIX_FMT_RGB24 && other->pixel_format() == AV_PIX_FMT_RGB24);
862 DCPOMATIC_ASSERT (position.x >= 0 && position.y >= 0);
864 int const N = min (position.x + other->size().width, size().width) - position.x;
865 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
866 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
867 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
868 memcpy (tp, op, N * 3);
874 Image::read_from_socket (shared_ptr<Socket> socket)
876 for (int i = 0; i < planes(); ++i) {
877 uint8_t* p = data()[i];
878 int const lines = sample_size(i).height;
879 for (int y = 0; y < lines; ++y) {
880 socket->read (p, line_size()[i]);
888 Image::write_to_socket (shared_ptr<Socket> socket) const
890 for (int i = 0; i < planes(); ++i) {
891 uint8_t* p = data()[i];
892 int const lines = sample_size(i).height;
893 for (int y = 0; y < lines; ++y) {
894 socket->write (p, line_size()[i]);
902 Image::bytes_per_pixel (int c) const
904 auto d = av_pix_fmt_desc_get(_pixel_format);
906 throw PixelFormatError ("bytes_per_pixel()", _pixel_format);
913 float bpp[4] = { 0, 0, 0, 0 };
915 #ifdef DCPOMATIC_HAVE_AVCOMPONENTDESCRIPTOR_DEPTH_MINUS1
916 bpp[0] = floor ((d->comp[0].depth_minus1 + 8) / 8);
917 if (d->nb_components > 1) {
918 bpp[1] = floor ((d->comp[1].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
920 if (d->nb_components > 2) {
921 bpp[2] = floor ((d->comp[2].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
923 if (d->nb_components > 3) {
924 bpp[3] = floor ((d->comp[3].depth_minus1 + 8) / 8) / pow (2.0f, d->log2_chroma_w);
927 bpp[0] = floor ((d->comp[0].depth + 7) / 8);
928 if (d->nb_components > 1) {
929 bpp[1] = floor ((d->comp[1].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
931 if (d->nb_components > 2) {
932 bpp[2] = floor ((d->comp[2].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
934 if (d->nb_components > 3) {
935 bpp[3] = floor ((d->comp[3].depth + 7) / 8) / pow (2.0f, d->log2_chroma_w);
939 if ((d->flags & AV_PIX_FMT_FLAG_PLANAR) == 0) {
940 /* Not planar; sum them up */
941 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
948 /** Construct a Image of a given size and format, allocating memory
951 * @param p Pixel format.
952 * @param s Size in pixels.
953 * @param alignment PADDED to make each row of this image aligned to a ALIGNMENT-byte boundary, otherwise COMPACT.
955 Image::Image (AVPixelFormat p, dcp::Size s, Alignment alignment)
958 , _alignment (alignment)
967 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
968 _data[0] = _data[1] = _data[2] = _data[3] = 0;
970 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
971 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
973 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
974 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
976 auto stride_round_up = [](int stride, int t) {
977 int const a = stride + (t - 1);
981 for (int i = 0; i < planes(); ++i) {
982 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
983 _stride[i] = stride_round_up (_line_size[i], _alignment == Alignment::PADDED ? ALIGNMENT : 1);
986 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
987 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
988 Hence on the last pixel of the last line it reads over the end of
989 the actual data by 1 byte. If the width of an image is a multiple
990 of the stride alignment there will be no padding at the end of image lines.
991 OS X crashes on this illegal read, though other operating systems don't
992 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
993 for that instruction to read safely.
995 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
996 over-reads by more then _avx. I can't follow the code to work out how much,
997 so I'll just over-allocate by ALIGNMENT bytes and have done with it. Empirical
998 testing suggests that it works.
1000 In addition to these concerns, we may read/write as much as a whole extra line
1001 at the end of each plane in cases where we are messing with offsets in order to
1002 do pad or crop. To solve this we over-allocate by an extra _stride[i] bytes.
1004 As an example: we may write to images starting at an offset so we get some padding.
1005 Hence we want to write in the following pattern:
1007 block start write start line end
1008 |..(padding)..|<------line-size------------->|..(padding)..|
1009 |..(padding)..|<------line-size------------->|..(padding)..|
1010 |..(padding)..|<------line-size------------->|..(padding)..|
1012 where line-size is of the smaller (inter_size) image and the full padded line length is that of
1013 out_size. To get things to work we have to tell FFmpeg that the stride is that of out_size.
1014 However some parts of FFmpeg (notably rgb48Toxyz12 in swscale.c) process data for the full
1015 specified *stride*. This does not matter until we get to the last line:
1017 block start write start line end
1018 |..(padding)..|<------line-size------------->|XXXwrittenXXX|
1019 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXX|
1020 |XXXwrittenXXX|<------line-size------------->|XXXwrittenXXXXXXwrittenXXX
1023 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
1024 #if HAVE_VALGRIND_MEMCHECK_H
1025 /* The data between the end of the line size and the stride is undefined but processed by
1026 libswscale, causing lots of valgrind errors. Mark it all defined to quell these errors.
1028 VALGRIND_MAKE_MEM_DEFINED (_data[i], _stride[i] * (sample_size(i).height + 1) + ALIGNMENT);
1034 Image::Image (Image const & other)
1035 : std::enable_shared_from_this<Image>(other)
1036 , _size (other._size)
1037 , _pixel_format (other._pixel_format)
1038 , _alignment (other._alignment)
1042 for (int i = 0; i < planes(); ++i) {
1043 uint8_t* p = _data[i];
1044 uint8_t* q = other._data[i];
1045 int const lines = sample_size(i).height;
1046 for (int j = 0; j < lines; ++j) {
1047 memcpy (p, q, _line_size[i]);
1049 q += other.stride()[i];
1055 Image::Image (AVFrame const * frame, Alignment alignment)
1056 : _size (frame->width, frame->height)
1057 , _pixel_format (static_cast<AVPixelFormat>(frame->format))
1058 , _alignment (alignment)
1060 DCPOMATIC_ASSERT (_pixel_format != AV_PIX_FMT_NONE);
1064 for (int i = 0; i < planes(); ++i) {
1065 uint8_t* p = _data[i];
1066 uint8_t* q = frame->data[i];
1067 int const lines = sample_size(i).height;
1068 for (int j = 0; j < lines; ++j) {
1069 memcpy (p, q, _line_size[i]);
1071 /* AVFrame's linesize is what we call `stride' */
1072 q += frame->linesize[i];
1078 Image::Image (shared_ptr<const Image> other, Alignment alignment)
1079 : _size (other->_size)
1080 , _pixel_format (other->_pixel_format)
1081 , _alignment (alignment)
1085 for (int i = 0; i < planes(); ++i) {
1086 DCPOMATIC_ASSERT (line_size()[i] == other->line_size()[i]);
1087 uint8_t* p = _data[i];
1088 uint8_t* q = other->data()[i];
1089 int const lines = sample_size(i).height;
1090 for (int j = 0; j < lines; ++j) {
1091 memcpy (p, q, line_size()[i]);
1093 q += other->stride()[i];
1100 Image::operator= (Image const & other)
1102 if (this == &other) {
1113 Image::swap (Image & other)
1115 std::swap (_size, other._size);
1116 std::swap (_pixel_format, other._pixel_format);
1118 for (int i = 0; i < 4; ++i) {
1119 std::swap (_data[i], other._data[i]);
1120 std::swap (_line_size[i], other._line_size[i]);
1121 std::swap (_stride[i], other._stride[i]);
1124 std::swap (_alignment, other._alignment);
1130 for (int i = 0; i < planes(); ++i) {
1135 av_free (_line_size);
1141 Image::data () const
1148 Image::line_size () const
1155 Image::stride () const
1162 Image::size () const
1169 Image::alignment () const
1176 merge (list<PositionImage> images, Image::Alignment alignment)
1178 if (images.empty ()) {
1182 if (images.size() == 1) {
1183 images.front().image = Image::ensure_alignment(images.front().image, alignment);
1184 return images.front();
1187 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
1188 for (auto const& i: images) {
1189 all.extend (dcpomatic::Rect<int>(i.position, i.image->size().width, i.image->size().height));
1192 auto merged = make_shared<Image>(images.front().image->pixel_format(), dcp::Size(all.width, all.height), alignment);
1193 merged->make_transparent ();
1194 for (auto const& i: images) {
1195 merged->alpha_blend (i.image, i.position - all.position());
1198 return PositionImage (merged, all.position ());
1203 operator== (Image const & a, Image const & b)
1205 if (a.planes() != b.planes() || a.pixel_format() != b.pixel_format() || a.alignment() != b.alignment()) {
1209 for (int c = 0; c < a.planes(); ++c) {
1210 if (a.sample_size(c).height != b.sample_size(c).height || a.line_size()[c] != b.line_size()[c] || a.stride()[c] != b.stride()[c]) {
1214 uint8_t* p = a.data()[c];
1215 uint8_t* q = b.data()[c];
1216 int const lines = a.sample_size(c).height;
1217 for (int y = 0; y < lines; ++y) {
1218 if (memcmp (p, q, a.line_size()[c]) != 0) {
1232 * @param f Amount to fade by; 0 is black, 1 is no fade.
1235 Image::fade (float f)
1237 /* U/V black value for 8-bit colour */
1238 static int const eight_bit_uv = (1 << 7) - 1;
1239 /* U/V black value for 10-bit colour */
1240 static uint16_t const ten_bit_uv = (1 << 9) - 1;
1242 switch (_pixel_format) {
1243 case AV_PIX_FMT_YUV420P:
1246 uint8_t* p = data()[0];
1247 int const lines = sample_size(0).height;
1248 for (int y = 0; y < lines; ++y) {
1250 for (int x = 0; x < line_size()[0]; ++x) {
1251 *q = int(float(*q) * f);
1258 for (int c = 1; c < 3; ++c) {
1259 uint8_t* p = data()[c];
1260 int const lines = sample_size(c).height;
1261 for (int y = 0; y < lines; ++y) {
1263 for (int x = 0; x < line_size()[c]; ++x) {
1264 *q = eight_bit_uv + int((int(*q) - eight_bit_uv) * f);
1274 case AV_PIX_FMT_RGB24:
1277 uint8_t* p = data()[0];
1278 int const lines = sample_size(0).height;
1279 for (int y = 0; y < lines; ++y) {
1281 for (int x = 0; x < line_size()[0]; ++x) {
1282 *q = int (float (*q) * f);
1290 case AV_PIX_FMT_XYZ12LE:
1291 case AV_PIX_FMT_RGB48LE:
1292 /* 16-bit little-endian */
1293 for (int c = 0; c < 3; ++c) {
1294 int const stride_pixels = stride()[c] / 2;
1295 int const line_size_pixels = line_size()[c] / 2;
1296 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1297 int const lines = sample_size(c).height;
1298 for (int y = 0; y < lines; ++y) {
1300 for (int x = 0; x < line_size_pixels; ++x) {
1301 *q = int (float (*q) * f);
1309 case AV_PIX_FMT_YUV422P10LE:
1313 int const stride_pixels = stride()[0] / 2;
1314 int const line_size_pixels = line_size()[0] / 2;
1315 uint16_t* p = reinterpret_cast<uint16_t*> (data()[0]);
1316 int const lines = sample_size(0).height;
1317 for (int y = 0; y < lines; ++y) {
1319 for (int x = 0; x < line_size_pixels; ++x) {
1320 *q = int(float(*q) * f);
1328 for (int c = 1; c < 3; ++c) {
1329 int const stride_pixels = stride()[c] / 2;
1330 int const line_size_pixels = line_size()[c] / 2;
1331 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
1332 int const lines = sample_size(c).height;
1333 for (int y = 0; y < lines; ++y) {
1335 for (int x = 0; x < line_size_pixels; ++x) {
1336 *q = ten_bit_uv + int((int(*q) - ten_bit_uv) * f);
1347 throw PixelFormatError ("fade()", _pixel_format);
1352 shared_ptr<const Image>
1353 Image::ensure_alignment (shared_ptr<const Image> image, Image::Alignment alignment)
1355 if (image->alignment() == alignment) {
1359 return make_shared<Image>(image, alignment);
1364 Image::memory_used () const
1367 for (int i = 0; i < planes(); ++i) {
1368 m += _stride[i] * sample_size(i).height;
1375 Image::video_range_to_full_range ()
1377 switch (_pixel_format) {
1378 case AV_PIX_FMT_RGB24:
1380 float const factor = 256.0 / 219.0;
1381 uint8_t* p = data()[0];
1382 int const lines = sample_size(0).height;
1383 for (int y = 0; y < lines; ++y) {
1385 for (int x = 0; x < line_size()[0]; ++x) {
1386 *q = clamp(lrintf((*q - 16) * factor), 0L, 255L);
1393 case AV_PIX_FMT_RGB48LE:
1395 float const factor = 65536.0 / 56064.0;
1396 uint16_t* p = reinterpret_cast<uint16_t*>(data()[0]);
1397 int const lines = sample_size(0).height;
1398 for (int y = 0; y < lines; ++y) {
1400 int const line_size_pixels = line_size()[0] / 2;
1401 for (int x = 0; x < line_size_pixels; ++x) {
1402 *q = clamp(lrintf((*q - 4096) * factor), 0L, 65535L);
1405 p += stride()[0] / 2;
1409 case AV_PIX_FMT_GBRP12LE:
1411 float const factor = 4096.0 / 3504.0;
1412 for (int c = 0; c < 3; ++c) {
1413 uint16_t* p = reinterpret_cast<uint16_t*>(data()[c]);
1414 int const lines = sample_size(c).height;
1415 for (int y = 0; y < lines; ++y) {
1417 int const line_size_pixels = line_size()[c] / 2;
1418 for (int x = 0; x < line_size_pixels; ++x) {
1419 *q = clamp(lrintf((*q - 256) * factor), 0L, 4095L);
1427 throw PixelFormatError ("video_range_to_full_range()", _pixel_format);