2 Copyright (C) 2012-2014 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 /** @file src/image.cc
21 * @brief A class to describe a video image.
26 #include <libswscale/swscale.h>
27 #include <libavutil/pixfmt.h>
28 #include <libavutil/pixdesc.h>
31 #include "exceptions.h"
35 #include "md5_digester.h"
44 using boost::shared_ptr;
48 Image::line_factor (int n) const
54 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
56 throw PixelFormatError ("lines()", _pixel_format);
59 return pow (2.0f, d->log2_chroma_h);
62 /** @param n Component index.
63 * @return Number of lines in the image for the given component.
66 Image::lines (int n) const
68 return rint (ceil (static_cast<double>(size().height) / line_factor (n)));
71 /** @return Number of components */
73 Image::components () const
75 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
77 throw PixelFormatError ("components()", _pixel_format);
80 if ((d->flags & PIX_FMT_PLANAR) == 0) {
84 return d->nb_components;
87 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size' */
89 Image::crop_scale_window (Crop crop, dcp::Size inter_size, dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
92 /* Empirical testing suggests that sws_scale() will crash if
93 the input image is not aligned.
97 assert (out_size.width >= inter_size.width);
98 assert (out_size.height >= inter_size.height);
100 /* Here's an image of out_size */
101 shared_ptr<Image> out (new Image (out_format, out_size, out_aligned));
104 /* Size of the image after any crop */
105 dcp::Size const cropped_size = crop.apply (size ());
107 /* Scale context for a scale from cropped_size to inter_size */
108 struct SwsContext* scale_context = sws_getContext (
109 cropped_size.width, cropped_size.height, pixel_format(),
110 inter_size.width, inter_size.height, out_format,
111 scaler->ffmpeg_id (), 0, 0, 0
114 if (!scale_context) {
115 throw StringError (N_("Could not allocate SwsContext"));
118 /* Prepare input data pointers with crop */
119 uint8_t* scale_in_data[components()];
120 for (int c = 0; c < components(); ++c) {
121 scale_in_data[c] = data()[c] + int (rint (bytes_per_pixel(c) * crop.left)) + stride()[c] * (crop.top / line_factor(c));
124 /* Corner of the image within out_size */
125 Position<int> const corner ((out_size.width - inter_size.width) / 2, (out_size.height - inter_size.height) / 2);
127 uint8_t* scale_out_data[out->components()];
128 for (int c = 0; c < out->components(); ++c) {
129 scale_out_data[c] = out->data()[c] + int (rint (out->bytes_per_pixel(c) * corner.x)) + out->stride()[c] * corner.y;
134 scale_in_data, stride(),
135 0, cropped_size.height,
136 scale_out_data, out->stride()
139 sws_freeContext (scale_context);
145 Image::scale (dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
148 /* Empirical testing suggests that sws_scale() will crash if
149 the input image is not aligned.
153 shared_ptr<Image> scaled (new Image (out_format, out_size, out_aligned));
155 struct SwsContext* scale_context = sws_getContext (
156 size().width, size().height, pixel_format(),
157 out_size.width, out_size.height, out_format,
158 scaler->ffmpeg_id (), 0, 0, 0
165 scaled->data(), scaled->stride()
168 sws_freeContext (scale_context);
174 Image::crop (Crop crop, bool aligned) const
176 dcp::Size cropped_size = crop.apply (size ());
177 shared_ptr<Image> out (new Image (pixel_format(), cropped_size, aligned));
179 for (int c = 0; c < components(); ++c) {
180 int const crop_left_in_bytes = bytes_per_pixel(c) * crop.left;
181 /* bytes_per_pixel() could be a fraction; in this case the stride will be rounded
182 up, and we need to make sure that we copy over the width (up to the stride)
183 rather than short of the width; hence the ceil() here.
185 int const cropped_width_in_bytes = ceil (bytes_per_pixel(c) * cropped_size.width);
187 /* Start of the source line, cropped from the top but not the left */
188 uint8_t* in_p = data()[c] + (crop.top / out->line_factor(c)) * stride()[c];
189 uint8_t* out_p = out->data()[c];
191 for (int y = 0; y < out->lines(c); ++y) {
192 memcpy (out_p, in_p + crop_left_in_bytes, cropped_width_in_bytes);
194 out_p += out->stride()[c];
201 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
203 Image::yuv_16_black (uint16_t v, bool alpha)
205 memset (data()[0], 0, lines(0) * stride()[0]);
206 for (int i = 1; i < 3; ++i) {
207 int16_t* p = reinterpret_cast<int16_t*> (data()[i]);
208 for (int y = 0; y < lines(i); ++y) {
209 /* We divide by 2 here because we are writing 2 bytes at a time */
210 for (int x = 0; x < line_size()[i] / 2; ++x) {
213 p += stride()[i] / 2;
218 memset (data()[3], 0, lines(3) * stride()[3]);
223 Image::swap_16 (uint16_t v)
225 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
231 /* U/V black value for 8-bit colour */
232 static uint8_t const eight_bit_uv = (1 << 7) - 1;
233 /* U/V black value for 9-bit colour */
234 static uint16_t const nine_bit_uv = (1 << 8) - 1;
235 /* U/V black value for 10-bit colour */
236 static uint16_t const ten_bit_uv = (1 << 9) - 1;
237 /* U/V black value for 16-bit colour */
238 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
240 switch (_pixel_format) {
241 case PIX_FMT_YUV420P:
242 case PIX_FMT_YUV422P:
243 case PIX_FMT_YUV444P:
244 case PIX_FMT_YUV411P:
245 memset (data()[0], 0, lines(0) * stride()[0]);
246 memset (data()[1], eight_bit_uv, lines(1) * stride()[1]);
247 memset (data()[2], eight_bit_uv, lines(2) * stride()[2]);
250 case PIX_FMT_YUVJ420P:
251 case PIX_FMT_YUVJ422P:
252 case PIX_FMT_YUVJ444P:
253 memset (data()[0], 0, lines(0) * stride()[0]);
254 memset (data()[1], eight_bit_uv + 1, lines(1) * stride()[1]);
255 memset (data()[2], eight_bit_uv + 1, lines(2) * stride()[2]);
258 case PIX_FMT_YUV422P9LE:
259 case PIX_FMT_YUV444P9LE:
260 yuv_16_black (nine_bit_uv, false);
263 case PIX_FMT_YUV422P9BE:
264 case PIX_FMT_YUV444P9BE:
265 yuv_16_black (swap_16 (nine_bit_uv), false);
268 case PIX_FMT_YUV422P10LE:
269 case PIX_FMT_YUV444P10LE:
270 yuv_16_black (ten_bit_uv, false);
273 case PIX_FMT_YUV422P16LE:
274 case PIX_FMT_YUV444P16LE:
275 yuv_16_black (sixteen_bit_uv, false);
278 case PIX_FMT_YUV444P10BE:
279 case PIX_FMT_YUV422P10BE:
280 yuv_16_black (swap_16 (ten_bit_uv), false);
283 case AV_PIX_FMT_YUVA420P9BE:
284 case AV_PIX_FMT_YUVA422P9BE:
285 case AV_PIX_FMT_YUVA444P9BE:
286 yuv_16_black (swap_16 (nine_bit_uv), true);
289 case AV_PIX_FMT_YUVA420P9LE:
290 case AV_PIX_FMT_YUVA422P9LE:
291 case AV_PIX_FMT_YUVA444P9LE:
292 yuv_16_black (nine_bit_uv, true);
295 case AV_PIX_FMT_YUVA420P10BE:
296 case AV_PIX_FMT_YUVA422P10BE:
297 case AV_PIX_FMT_YUVA444P10BE:
298 yuv_16_black (swap_16 (ten_bit_uv), true);
301 case AV_PIX_FMT_YUVA420P10LE:
302 case AV_PIX_FMT_YUVA422P10LE:
303 case AV_PIX_FMT_YUVA444P10LE:
304 yuv_16_black (ten_bit_uv, true);
307 case AV_PIX_FMT_YUVA420P16BE:
308 case AV_PIX_FMT_YUVA422P16BE:
309 case AV_PIX_FMT_YUVA444P16BE:
310 yuv_16_black (swap_16 (sixteen_bit_uv), true);
313 case AV_PIX_FMT_YUVA420P16LE:
314 case AV_PIX_FMT_YUVA422P16LE:
315 case AV_PIX_FMT_YUVA444P16LE:
316 yuv_16_black (sixteen_bit_uv, true);
324 case PIX_FMT_RGB555LE:
325 memset (data()[0], 0, lines(0) * stride()[0]);
328 case PIX_FMT_UYVY422:
330 int const Y = lines(0);
331 int const X = line_size()[0];
332 uint8_t* p = data()[0];
333 for (int y = 0; y < Y; ++y) {
334 for (int x = 0; x < X / 4; ++x) {
335 *p++ = eight_bit_uv; // Cb
337 *p++ = eight_bit_uv; // Cr
345 throw PixelFormatError ("make_black()", _pixel_format);
350 Image::make_transparent ()
352 if (_pixel_format != PIX_FMT_RGBA) {
353 throw PixelFormatError ("make_transparent()", _pixel_format);
356 memset (data()[0], 0, lines(0) * stride()[0]);
360 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
362 assert (other->pixel_format() == PIX_FMT_RGBA);
363 int const other_bpp = 4;
366 switch (_pixel_format) {
378 int start_tx = position.x;
382 start_ox = -start_tx;
386 int start_ty = position.y;
390 start_oy = -start_ty;
394 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
395 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
396 uint8_t* op = other->data()[0] + oy * other->stride()[0];
397 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
398 float const alpha = float (op[3]) / 255;
399 tp[0] = op[0] + (tp[0] * (1 - alpha));
400 tp[1] = op[1] + (tp[1] * (1 - alpha));
401 tp[2] = op[2] + (tp[2] * (1 - alpha));
402 tp[3] = op[3] + (tp[3] * (1 - alpha));
411 Image::copy (shared_ptr<const Image> other, Position<int> position)
413 /* Only implemented for RGB24 onto RGB24 so far */
414 assert (_pixel_format == PIX_FMT_RGB24 && other->pixel_format() == PIX_FMT_RGB24);
415 assert (position.x >= 0 && position.y >= 0);
417 int const N = min (position.x + other->size().width, size().width) - position.x;
418 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
419 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
420 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
421 memcpy (tp, op, N * 3);
426 Image::read_from_socket (shared_ptr<Socket> socket)
428 for (int i = 0; i < components(); ++i) {
429 uint8_t* p = data()[i];
430 for (int y = 0; y < lines(i); ++y) {
431 socket->read (p, line_size()[i]);
438 Image::write_to_socket (shared_ptr<Socket> socket) const
440 for (int i = 0; i < components(); ++i) {
441 uint8_t* p = data()[i];
442 for (int y = 0; y < lines(i); ++y) {
443 socket->write (p, line_size()[i]);
451 Image::bytes_per_pixel (int c) const
453 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
455 throw PixelFormatError ("lines()", _pixel_format);
458 if (c >= components()) {
462 float bpp[4] = { 0, 0, 0, 0 };
464 bpp[0] = floor ((d->comp[0].depth_minus1 + 1 + 7) / 8);
465 if (d->nb_components > 1) {
466 bpp[1] = floor ((d->comp[1].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
468 if (d->nb_components > 2) {
469 bpp[2] = floor ((d->comp[2].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
471 if (d->nb_components > 3) {
472 bpp[3] = floor ((d->comp[3].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
475 if ((d->flags & PIX_FMT_PLANAR) == 0) {
476 /* Not planar; sum them up */
477 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
483 /** Construct a Image of a given size and format, allocating memory
486 * @param p Pixel format.
487 * @param s Size in pixels.
489 Image::Image (AVPixelFormat p, dcp::Size s, bool aligned)
500 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
501 _data[0] = _data[1] = _data[2] = _data[3] = 0;
503 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
504 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
506 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
507 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
509 for (int i = 0; i < components(); ++i) {
510 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
511 _stride[i] = stride_round_up (i, _line_size, _aligned ? 32 : 1);
513 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
514 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
515 Hence on the last pixel of the last line it reads over the end of
516 the actual data by 1 byte. If the width of an image is a multiple
517 of the stride alignment there will be no padding at the end of image lines.
518 OS X crashes on this illegal read, though other operating systems don't
519 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
520 for that instruction to read safely.
522 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
523 over-reads by more then _avx. I can't follow the code to work out how much,
524 so I'll just over-allocate by 32 bytes and have done with it. Empirical
525 testing suggests that it works.
527 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * lines (i) + 32);
531 Image::Image (Image const & other)
533 , _pixel_format (other._pixel_format)
534 , _aligned (other._aligned)
538 for (int i = 0; i < components(); ++i) {
539 uint8_t* p = _data[i];
540 uint8_t* q = other._data[i];
541 for (int j = 0; j < lines(i); ++j) {
542 memcpy (p, q, _line_size[i]);
544 q += other.stride()[i];
549 Image::Image (AVFrame* frame)
550 : dcp::Image (dcp::Size (frame->width, frame->height))
551 , _pixel_format (static_cast<AVPixelFormat> (frame->format))
556 for (int i = 0; i < components(); ++i) {
557 uint8_t* p = _data[i];
558 uint8_t* q = frame->data[i];
559 for (int j = 0; j < lines(i); ++j) {
560 memcpy (p, q, _line_size[i]);
562 /* AVFrame's linesize is what we call `stride' */
563 q += frame->linesize[i];
568 Image::Image (shared_ptr<const Image> other, bool aligned)
570 , _pixel_format (other->_pixel_format)
575 for (int i = 0; i < components(); ++i) {
576 assert(line_size()[i] == other->line_size()[i]);
577 uint8_t* p = _data[i];
578 uint8_t* q = other->data()[i];
579 for (int j = 0; j < lines(i); ++j) {
580 memcpy (p, q, line_size()[i]);
582 q += other->stride()[i];
588 Image::operator= (Image const & other)
590 if (this == &other) {
600 Image::swap (Image & other)
602 dcp::Image::swap (other);
604 std::swap (_pixel_format, other._pixel_format);
606 for (int i = 0; i < 4; ++i) {
607 std::swap (_data[i], other._data[i]);
608 std::swap (_line_size[i], other._line_size[i]);
609 std::swap (_stride[i], other._stride[i]);
612 std::swap (_aligned, other._aligned);
615 /** Destroy a Image */
618 for (int i = 0; i < components(); ++i) {
623 av_free (_line_size);
634 Image::line_size () const
640 Image::stride () const
652 Image::aligned () const
658 merge (list<PositionImage> images)
660 if (images.empty ()) {
661 return PositionImage ();
664 if (images.size() == 1) {
665 return images.front ();
668 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
669 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
670 all.extend (dcpomatic::Rect<int> (i->position, i->image->size().width, i->image->size().height));
673 shared_ptr<Image> merged (new Image (images.front().image->pixel_format (), dcp::Size (all.width, all.height), true));
674 merged->make_transparent ();
675 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
676 merged->alpha_blend (i->image, i->position - all.position());
679 return PositionImage (merged, all.position ());
683 Image::digest () const
685 MD5Digester digester;
687 for (int i = 0; i < components(); ++i) {
688 digester.add (data()[i], line_size()[i]);
691 return digester.get ();
695 operator== (Image const & a, Image const & b)
697 if (a.components() != b.components() || a.pixel_format() != b.pixel_format() || a.aligned() != b.aligned()) {
701 for (int c = 0; c < a.components(); ++c) {
702 if (a.lines(c) != b.lines(c) || a.line_size()[c] != b.line_size()[c] || a.stride()[c] != b.stride()[c]) {
706 uint8_t* p = a.data()[c];
707 uint8_t* q = b.data()[c];
708 for (int y = 0; y < a.lines(c); ++y) {
709 if (memcmp (p, q, a.line_size()[c]) != 0) {