2 Copyright (C) 2012-2015 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 /** @file src/image.cc
21 * @brief A class to describe a video image.
25 #include "exceptions.h"
30 #include "md5_digester.h"
31 #include "dcpomatic_socket.h"
33 #include <libswscale/swscale.h>
34 #include <libavutil/pixfmt.h>
35 #include <libavutil/pixdesc.h>
46 using boost::shared_ptr;
50 Image::line_factor (int n) const
56 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
58 throw PixelFormatError ("lines()", _pixel_format);
61 return pow (2.0f, d->log2_chroma_h);
64 /** @param n Component index.
65 * @return Number of lines in the image for the given component.
68 Image::lines (int n) const
70 return rint (ceil (static_cast<double>(size().height) / line_factor (n)));
73 /** @return Number of components */
75 Image::components () const
77 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
79 throw PixelFormatError ("components()", _pixel_format);
82 if ((d->flags & PIX_FMT_PLANAR) == 0) {
86 return d->nb_components;
89 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size' */
91 Image::crop_scale_window (Crop crop, dcp::Size inter_size, dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
93 DCPOMATIC_ASSERT (scaler);
94 /* Empirical testing suggests that sws_scale() will crash if
95 the input image is not aligned.
97 DCPOMATIC_ASSERT (aligned ());
99 DCPOMATIC_ASSERT (out_size.width >= inter_size.width);
100 DCPOMATIC_ASSERT (out_size.height >= inter_size.height);
102 /* Here's an image of out_size */
103 shared_ptr<Image> out (new Image (out_format, out_size, out_aligned));
106 /* Size of the image after any crop */
107 dcp::Size const cropped_size = crop.apply (size ());
109 /* Scale context for a scale from cropped_size to inter_size */
110 struct SwsContext* scale_context = sws_getContext (
111 cropped_size.width, cropped_size.height, pixel_format(),
112 inter_size.width, inter_size.height, out_format,
113 scaler->ffmpeg_id (), 0, 0, 0
116 if (!scale_context) {
117 throw StringError (N_("Could not allocate SwsContext"));
120 /* Prepare input data pointers with crop */
121 uint8_t* scale_in_data[components()];
122 for (int c = 0; c < components(); ++c) {
123 scale_in_data[c] = data()[c] + int (rint (bytes_per_pixel(c) * crop.left)) + stride()[c] * (crop.top / line_factor(c));
126 /* Corner of the image within out_size */
127 Position<int> const corner ((out_size.width - inter_size.width) / 2, (out_size.height - inter_size.height) / 2);
129 uint8_t* scale_out_data[out->components()];
130 for (int c = 0; c < out->components(); ++c) {
131 scale_out_data[c] = out->data()[c] + int (rint (out->bytes_per_pixel(c) * corner.x)) + out->stride()[c] * corner.y;
136 scale_in_data, stride(),
137 0, cropped_size.height,
138 scale_out_data, out->stride()
141 sws_freeContext (scale_context);
147 Image::scale (dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
149 DCPOMATIC_ASSERT (scaler);
150 /* Empirical testing suggests that sws_scale() will crash if
151 the input image is not aligned.
153 DCPOMATIC_ASSERT (aligned ());
155 shared_ptr<Image> scaled (new Image (out_format, out_size, out_aligned));
157 struct SwsContext* scale_context = sws_getContext (
158 size().width, size().height, pixel_format(),
159 out_size.width, out_size.height, out_format,
160 scaler->ffmpeg_id (), 0, 0, 0
167 scaled->data(), scaled->stride()
170 sws_freeContext (scale_context);
176 Image::crop (Crop crop, bool aligned) const
178 dcp::Size cropped_size = crop.apply (size ());
179 shared_ptr<Image> out (new Image (pixel_format(), cropped_size, aligned));
181 for (int c = 0; c < components(); ++c) {
182 int const crop_left_in_bytes = bytes_per_pixel(c) * crop.left;
183 /* bytes_per_pixel() could be a fraction; in this case the stride will be rounded
184 up, and we need to make sure that we copy over the width (up to the stride)
185 rather than short of the width; hence the ceil() here.
187 int const cropped_width_in_bytes = ceil (bytes_per_pixel(c) * cropped_size.width);
189 /* Start of the source line, cropped from the top but not the left */
190 uint8_t* in_p = data()[c] + (crop.top / out->line_factor(c)) * stride()[c];
191 uint8_t* out_p = out->data()[c];
193 for (int y = 0; y < out->lines(c); ++y) {
194 memcpy (out_p, in_p + crop_left_in_bytes, cropped_width_in_bytes);
196 out_p += out->stride()[c];
203 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
205 Image::yuv_16_black (uint16_t v, bool alpha)
207 memset (data()[0], 0, lines(0) * stride()[0]);
208 for (int i = 1; i < 3; ++i) {
209 int16_t* p = reinterpret_cast<int16_t*> (data()[i]);
210 for (int y = 0; y < lines(i); ++y) {
211 /* We divide by 2 here because we are writing 2 bytes at a time */
212 for (int x = 0; x < line_size()[i] / 2; ++x) {
215 p += stride()[i] / 2;
220 memset (data()[3], 0, lines(3) * stride()[3]);
225 Image::swap_16 (uint16_t v)
227 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
233 /* U/V black value for 8-bit colour */
234 static uint8_t const eight_bit_uv = (1 << 7) - 1;
235 /* U/V black value for 9-bit colour */
236 static uint16_t const nine_bit_uv = (1 << 8) - 1;
237 /* U/V black value for 10-bit colour */
238 static uint16_t const ten_bit_uv = (1 << 9) - 1;
239 /* U/V black value for 16-bit colour */
240 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
242 switch (_pixel_format) {
243 case PIX_FMT_YUV420P:
244 case PIX_FMT_YUV422P:
245 case PIX_FMT_YUV444P:
246 case PIX_FMT_YUV411P:
247 memset (data()[0], 0, lines(0) * stride()[0]);
248 memset (data()[1], eight_bit_uv, lines(1) * stride()[1]);
249 memset (data()[2], eight_bit_uv, lines(2) * stride()[2]);
252 case PIX_FMT_YUVJ420P:
253 case PIX_FMT_YUVJ422P:
254 case PIX_FMT_YUVJ444P:
255 memset (data()[0], 0, lines(0) * stride()[0]);
256 memset (data()[1], eight_bit_uv + 1, lines(1) * stride()[1]);
257 memset (data()[2], eight_bit_uv + 1, lines(2) * stride()[2]);
260 case PIX_FMT_YUV422P9LE:
261 case PIX_FMT_YUV444P9LE:
262 yuv_16_black (nine_bit_uv, false);
265 case PIX_FMT_YUV422P9BE:
266 case PIX_FMT_YUV444P9BE:
267 yuv_16_black (swap_16 (nine_bit_uv), false);
270 case PIX_FMT_YUV422P10LE:
271 case PIX_FMT_YUV444P10LE:
272 yuv_16_black (ten_bit_uv, false);
275 case PIX_FMT_YUV422P16LE:
276 case PIX_FMT_YUV444P16LE:
277 yuv_16_black (sixteen_bit_uv, false);
280 case PIX_FMT_YUV444P10BE:
281 case PIX_FMT_YUV422P10BE:
282 yuv_16_black (swap_16 (ten_bit_uv), false);
285 case AV_PIX_FMT_YUVA420P9BE:
286 case AV_PIX_FMT_YUVA422P9BE:
287 case AV_PIX_FMT_YUVA444P9BE:
288 yuv_16_black (swap_16 (nine_bit_uv), true);
291 case AV_PIX_FMT_YUVA420P9LE:
292 case AV_PIX_FMT_YUVA422P9LE:
293 case AV_PIX_FMT_YUVA444P9LE:
294 yuv_16_black (nine_bit_uv, true);
297 case AV_PIX_FMT_YUVA420P10BE:
298 case AV_PIX_FMT_YUVA422P10BE:
299 case AV_PIX_FMT_YUVA444P10BE:
300 yuv_16_black (swap_16 (ten_bit_uv), true);
303 case AV_PIX_FMT_YUVA420P10LE:
304 case AV_PIX_FMT_YUVA422P10LE:
305 case AV_PIX_FMT_YUVA444P10LE:
306 yuv_16_black (ten_bit_uv, true);
309 case AV_PIX_FMT_YUVA420P16BE:
310 case AV_PIX_FMT_YUVA422P16BE:
311 case AV_PIX_FMT_YUVA444P16BE:
312 yuv_16_black (swap_16 (sixteen_bit_uv), true);
315 case AV_PIX_FMT_YUVA420P16LE:
316 case AV_PIX_FMT_YUVA422P16LE:
317 case AV_PIX_FMT_YUVA444P16LE:
318 yuv_16_black (sixteen_bit_uv, true);
326 case PIX_FMT_RGB555LE:
327 case PIX_FMT_RGB48LE:
328 case PIX_FMT_RGB48BE:
329 memset (data()[0], 0, lines(0) * stride()[0]);
332 case PIX_FMT_UYVY422:
334 int const Y = lines(0);
335 int const X = line_size()[0];
336 uint8_t* p = data()[0];
337 for (int y = 0; y < Y; ++y) {
338 for (int x = 0; x < X / 4; ++x) {
339 *p++ = eight_bit_uv; // Cb
341 *p++ = eight_bit_uv; // Cr
349 throw PixelFormatError ("make_black()", _pixel_format);
354 Image::make_transparent ()
356 if (_pixel_format != PIX_FMT_RGBA) {
357 throw PixelFormatError ("make_transparent()", _pixel_format);
360 memset (data()[0], 0, lines(0) * stride()[0]);
364 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
366 DCPOMATIC_ASSERT (other->pixel_format() == PIX_FMT_RGBA);
367 int const other_bpp = 4;
369 int start_tx = position.x;
373 start_ox = -start_tx;
377 int start_ty = position.y;
381 start_oy = -start_ty;
385 switch (_pixel_format) {
388 int const this_bpp = 3;
389 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
390 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
391 uint8_t* op = other->data()[0] + oy * other->stride()[0];
392 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
393 float const alpha = float (op[3]) / 255;
394 tp[0] = op[0] * alpha + tp[0] * (1 - alpha);
395 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
396 tp[2] = op[2] * alpha + tp[2] * (1 - alpha);
407 int const this_bpp = 4;
408 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
409 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
410 uint8_t* op = other->data()[0] + oy * other->stride()[0];
411 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
412 float const alpha = float (op[3]) / 255;
413 tp[0] = op[0] * alpha + tp[0] * (1 - alpha);
414 tp[1] = op[1] * alpha + tp[1] * (1 - alpha);
415 tp[2] = op[2] * alpha + tp[2] * (1 - alpha);
416 tp[3] = op[3] * alpha + tp[3] * (1 - alpha);
424 case PIX_FMT_RGB48LE:
426 int const this_bpp = 6;
427 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
428 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
429 uint8_t* op = other->data()[0] + oy * other->stride()[0];
430 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
431 float const alpha = float (op[3]) / 255;
432 /* Blend high bytes */
433 tp[1] = op[0] * alpha + tp[1] * (1 - alpha);
434 tp[3] = op[1] * alpha + tp[3] * (1 - alpha);
435 tp[5] = op[2] * alpha + tp[5] * (1 - alpha);
444 DCPOMATIC_ASSERT (false);
449 Image::copy (shared_ptr<const Image> other, Position<int> position)
451 /* Only implemented for RGB24 onto RGB24 so far */
452 DCPOMATIC_ASSERT (_pixel_format == PIX_FMT_RGB24 && other->pixel_format() == PIX_FMT_RGB24);
453 DCPOMATIC_ASSERT (position.x >= 0 && position.y >= 0);
455 int const N = min (position.x + other->size().width, size().width) - position.x;
456 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
457 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
458 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
459 memcpy (tp, op, N * 3);
464 Image::read_from_socket (shared_ptr<Socket> socket)
466 for (int i = 0; i < components(); ++i) {
467 uint8_t* p = data()[i];
468 for (int y = 0; y < lines(i); ++y) {
469 socket->read (p, line_size()[i]);
476 Image::write_to_socket (shared_ptr<Socket> socket) const
478 for (int i = 0; i < components(); ++i) {
479 uint8_t* p = data()[i];
480 for (int y = 0; y < lines(i); ++y) {
481 socket->write (p, line_size()[i]);
489 Image::bytes_per_pixel (int c) const
491 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
493 throw PixelFormatError ("lines()", _pixel_format);
496 if (c >= components()) {
500 float bpp[4] = { 0, 0, 0, 0 };
502 bpp[0] = floor ((d->comp[0].depth_minus1 + 1 + 7) / 8);
503 if (d->nb_components > 1) {
504 bpp[1] = floor ((d->comp[1].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
506 if (d->nb_components > 2) {
507 bpp[2] = floor ((d->comp[2].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
509 if (d->nb_components > 3) {
510 bpp[3] = floor ((d->comp[3].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
513 if ((d->flags & PIX_FMT_PLANAR) == 0) {
514 /* Not planar; sum them up */
515 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
521 /** Construct a Image of a given size and format, allocating memory
524 * @param p Pixel format.
525 * @param s Size in pixels.
527 Image::Image (AVPixelFormat p, dcp::Size s, bool aligned)
538 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
539 _data[0] = _data[1] = _data[2] = _data[3] = 0;
541 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
542 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
544 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
545 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
547 for (int i = 0; i < components(); ++i) {
548 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
549 _stride[i] = stride_round_up (i, _line_size, _aligned ? 32 : 1);
551 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
552 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
553 Hence on the last pixel of the last line it reads over the end of
554 the actual data by 1 byte. If the width of an image is a multiple
555 of the stride alignment there will be no padding at the end of image lines.
556 OS X crashes on this illegal read, though other operating systems don't
557 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
558 for that instruction to read safely.
560 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
561 over-reads by more then _avx. I can't follow the code to work out how much,
562 so I'll just over-allocate by 32 bytes and have done with it. Empirical
563 testing suggests that it works.
565 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * lines (i) + 32);
569 Image::Image (Image const & other)
570 : _size (other._size)
571 , _pixel_format (other._pixel_format)
572 , _aligned (other._aligned)
576 for (int i = 0; i < components(); ++i) {
577 uint8_t* p = _data[i];
578 uint8_t* q = other._data[i];
579 for (int j = 0; j < lines(i); ++j) {
580 memcpy (p, q, _line_size[i]);
582 q += other.stride()[i];
587 Image::Image (AVFrame* frame)
588 : _size (frame->width, frame->height)
589 , _pixel_format (static_cast<AVPixelFormat> (frame->format))
594 for (int i = 0; i < components(); ++i) {
595 uint8_t* p = _data[i];
596 uint8_t* q = frame->data[i];
597 for (int j = 0; j < lines(i); ++j) {
598 memcpy (p, q, _line_size[i]);
600 /* AVFrame's linesize is what we call `stride' */
601 q += frame->linesize[i];
606 Image::Image (shared_ptr<const Image> other, bool aligned)
607 : _size (other->_size)
608 , _pixel_format (other->_pixel_format)
613 for (int i = 0; i < components(); ++i) {
614 DCPOMATIC_ASSERT (line_size()[i] == other->line_size()[i]);
615 uint8_t* p = _data[i];
616 uint8_t* q = other->data()[i];
617 for (int j = 0; j < lines(i); ++j) {
618 memcpy (p, q, line_size()[i]);
620 q += other->stride()[i];
626 Image::operator= (Image const & other)
628 if (this == &other) {
638 Image::swap (Image & other)
640 std::swap (_size, other._size);
641 std::swap (_pixel_format, other._pixel_format);
643 for (int i = 0; i < 4; ++i) {
644 std::swap (_data[i], other._data[i]);
645 std::swap (_line_size[i], other._line_size[i]);
646 std::swap (_stride[i], other._stride[i]);
649 std::swap (_aligned, other._aligned);
652 /** Destroy a Image */
655 for (int i = 0; i < components(); ++i) {
660 av_free (_line_size);
671 Image::line_size () const
677 Image::stride () const
689 Image::aligned () const
695 merge (list<PositionImage> images)
697 if (images.empty ()) {
698 return PositionImage ();
701 if (images.size() == 1) {
702 return images.front ();
705 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
706 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
707 all.extend (dcpomatic::Rect<int> (i->position, i->image->size().width, i->image->size().height));
710 shared_ptr<Image> merged (new Image (images.front().image->pixel_format (), dcp::Size (all.width, all.height), true));
711 merged->make_transparent ();
712 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
713 merged->alpha_blend (i->image, i->position - all.position());
716 return PositionImage (merged, all.position ());
720 Image::digest () const
722 MD5Digester digester;
724 for (int i = 0; i < components(); ++i) {
725 digester.add (data()[i], line_size()[i]);
728 return digester.get ();
732 operator== (Image const & a, Image const & b)
734 if (a.components() != b.components() || a.pixel_format() != b.pixel_format() || a.aligned() != b.aligned()) {
738 for (int c = 0; c < a.components(); ++c) {
739 if (a.lines(c) != b.lines(c) || a.line_size()[c] != b.line_size()[c] || a.stride()[c] != b.stride()[c]) {
743 uint8_t* p = a.data()[c];
744 uint8_t* q = b.data()[c];
745 for (int y = 0; y < a.lines(c); ++y) {
746 if (memcmp (p, q, a.line_size()[c]) != 0) {
759 Image::fade (float f)
761 switch (_pixel_format) {
762 case PIX_FMT_YUV420P:
763 case PIX_FMT_YUV422P:
764 case PIX_FMT_YUV444P:
765 case PIX_FMT_YUV411P:
766 case PIX_FMT_YUVJ420P:
767 case PIX_FMT_YUVJ422P:
768 case PIX_FMT_YUVJ444P:
774 case PIX_FMT_RGB555LE:
776 for (int c = 0; c < 3; ++c) {
777 uint8_t* p = data()[c];
778 for (int y = 0; y < lines(c); ++y) {
780 for (int x = 0; x < line_size()[c]; ++x) {
781 *q = int (float (*q) * f);
789 case PIX_FMT_YUV422P9LE:
790 case PIX_FMT_YUV444P9LE:
791 case PIX_FMT_YUV422P10LE:
792 case PIX_FMT_YUV444P10LE:
793 case PIX_FMT_YUV422P16LE:
794 case PIX_FMT_YUV444P16LE:
795 case AV_PIX_FMT_YUVA420P9LE:
796 case AV_PIX_FMT_YUVA422P9LE:
797 case AV_PIX_FMT_YUVA444P9LE:
798 case AV_PIX_FMT_YUVA420P10LE:
799 case AV_PIX_FMT_YUVA422P10LE:
800 case AV_PIX_FMT_YUVA444P10LE:
801 /* 16-bit little-endian */
802 for (int c = 0; c < 3; ++c) {
803 int const stride_pixels = stride()[c] / 2;
804 int const line_size_pixels = line_size()[c] / 2;
805 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
806 for (int y = 0; y < lines(c); ++y) {
808 for (int x = 0; x < line_size_pixels; ++x) {
809 *q = int (float (*q) * f);
817 case PIX_FMT_YUV422P9BE:
818 case PIX_FMT_YUV444P9BE:
819 case PIX_FMT_YUV444P10BE:
820 case PIX_FMT_YUV422P10BE:
821 case AV_PIX_FMT_YUVA420P9BE:
822 case AV_PIX_FMT_YUVA422P9BE:
823 case AV_PIX_FMT_YUVA444P9BE:
824 case AV_PIX_FMT_YUVA420P10BE:
825 case AV_PIX_FMT_YUVA422P10BE:
826 case AV_PIX_FMT_YUVA444P10BE:
827 case AV_PIX_FMT_YUVA420P16BE:
828 case AV_PIX_FMT_YUVA422P16BE:
829 case AV_PIX_FMT_YUVA444P16BE:
830 /* 16-bit big-endian */
831 for (int c = 0; c < 3; ++c) {
832 int const stride_pixels = stride()[c] / 2;
833 int const line_size_pixels = line_size()[c] / 2;
834 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
835 for (int y = 0; y < lines(c); ++y) {
837 for (int x = 0; x < line_size_pixels; ++x) {
838 *q = swap_16 (int (float (swap_16 (*q)) * f));
846 case PIX_FMT_UYVY422:
848 int const Y = lines(0);
849 int const X = line_size()[0];
850 uint8_t* p = data()[0];
851 for (int y = 0; y < Y; ++y) {
852 for (int x = 0; x < X; ++x) {
853 *p = int (float (*p) * f);
861 throw PixelFormatError ("fade()", _pixel_format);