2 Copyright (C) 2012-2014 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 /** @file src/image.cc
21 * @brief A class to describe a video image.
26 #include <libswscale/swscale.h>
27 #include <libavutil/pixfmt.h>
28 #include <libavutil/pixdesc.h>
31 #include "exceptions.h"
35 #include "md5_digester.h"
44 using std::stringstream;
45 using boost::shared_ptr;
49 Image::line_factor (int n) const
55 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
57 throw PixelFormatError ("lines()", _pixel_format);
60 return pow (2.0f, d->log2_chroma_h);
63 /** @param n Component index.
64 * @return Number of lines in the image for the given component.
67 Image::lines (int n) const
69 return rint (ceil (static_cast<double>(size().height) / line_factor (n)));
72 /** @return Number of components */
74 Image::components () const
76 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
78 throw PixelFormatError ("components()", _pixel_format);
81 if ((d->flags & PIX_FMT_PLANAR) == 0) {
85 return d->nb_components;
88 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size' */
90 Image::crop_scale_window (Crop crop, dcp::Size inter_size, dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
93 /* Empirical testing suggests that sws_scale() will crash if
94 the input image is not aligned.
98 assert (out_size.width >= inter_size.width);
99 assert (out_size.height >= inter_size.height);
101 /* Here's an image of out_size */
102 shared_ptr<Image> out (new Image (out_format, out_size, out_aligned));
105 /* Size of the image after any crop */
106 dcp::Size const cropped_size = crop.apply (size ());
108 /* Scale context for a scale from cropped_size to inter_size */
109 struct SwsContext* scale_context = sws_getContext (
110 cropped_size.width, cropped_size.height, pixel_format(),
111 inter_size.width, inter_size.height, out_format,
112 scaler->ffmpeg_id (), 0, 0, 0
115 if (!scale_context) {
116 throw StringError (N_("Could not allocate SwsContext"));
119 /* Prepare input data pointers with crop */
120 uint8_t* scale_in_data[components()];
121 for (int c = 0; c < components(); ++c) {
122 scale_in_data[c] = data()[c] + int (rint (bytes_per_pixel(c) * crop.left)) + stride()[c] * (crop.top / line_factor(c));
125 /* Corner of the image within out_size */
126 Position<int> const corner ((out_size.width - inter_size.width) / 2, (out_size.height - inter_size.height) / 2);
128 uint8_t* scale_out_data[out->components()];
129 for (int c = 0; c < out->components(); ++c) {
130 scale_out_data[c] = out->data()[c] + int (rint (out->bytes_per_pixel(c) * corner.x)) + out->stride()[c] * corner.y;
135 scale_in_data, stride(),
136 0, cropped_size.height,
137 scale_out_data, out->stride()
140 sws_freeContext (scale_context);
146 Image::scale (dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
149 /* Empirical testing suggests that sws_scale() will crash if
150 the input image is not aligned.
154 shared_ptr<Image> scaled (new Image (out_format, out_size, out_aligned));
156 struct SwsContext* scale_context = sws_getContext (
157 size().width, size().height, pixel_format(),
158 out_size.width, out_size.height, out_format,
159 scaler->ffmpeg_id (), 0, 0, 0
166 scaled->data(), scaled->stride()
169 sws_freeContext (scale_context);
175 Image::crop (Crop crop, bool aligned) const
177 dcp::Size cropped_size = crop.apply (size ());
178 shared_ptr<Image> out (new Image (pixel_format(), cropped_size, aligned));
180 for (int c = 0; c < components(); ++c) {
181 int const crop_left_in_bytes = bytes_per_pixel(c) * crop.left;
182 /* bytes_per_pixel() could be a fraction; in this case the stride will be rounded
183 up, and we need to make sure that we copy over the width (up to the stride)
184 rather than short of the width; hence the ceil() here.
186 int const cropped_width_in_bytes = ceil (bytes_per_pixel(c) * cropped_size.width);
188 /* Start of the source line, cropped from the top but not the left */
189 uint8_t* in_p = data()[c] + (crop.top / out->line_factor(c)) * stride()[c];
190 uint8_t* out_p = out->data()[c];
192 for (int y = 0; y < out->lines(c); ++y) {
193 memcpy (out_p, in_p + crop_left_in_bytes, cropped_width_in_bytes);
195 out_p += out->stride()[c];
202 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
204 Image::yuv_16_black (uint16_t v, bool alpha)
206 memset (data()[0], 0, lines(0) * stride()[0]);
207 for (int i = 1; i < 3; ++i) {
208 int16_t* p = reinterpret_cast<int16_t*> (data()[i]);
209 for (int y = 0; y < lines(i); ++y) {
210 /* We divide by 2 here because we are writing 2 bytes at a time */
211 for (int x = 0; x < line_size()[i] / 2; ++x) {
214 p += stride()[i] / 2;
219 memset (data()[3], 0, lines(3) * stride()[3]);
224 Image::swap_16 (uint16_t v)
226 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
232 /* U/V black value for 8-bit colour */
233 static uint8_t const eight_bit_uv = (1 << 7) - 1;
234 /* U/V black value for 9-bit colour */
235 static uint16_t const nine_bit_uv = (1 << 8) - 1;
236 /* U/V black value for 10-bit colour */
237 static uint16_t const ten_bit_uv = (1 << 9) - 1;
238 /* U/V black value for 16-bit colour */
239 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
241 switch (_pixel_format) {
242 case PIX_FMT_YUV420P:
243 case PIX_FMT_YUV422P:
244 case PIX_FMT_YUV444P:
245 case PIX_FMT_YUV411P:
246 memset (data()[0], 0, lines(0) * stride()[0]);
247 memset (data()[1], eight_bit_uv, lines(1) * stride()[1]);
248 memset (data()[2], eight_bit_uv, lines(2) * stride()[2]);
251 case PIX_FMT_YUVJ420P:
252 case PIX_FMT_YUVJ422P:
253 case PIX_FMT_YUVJ444P:
254 memset (data()[0], 0, lines(0) * stride()[0]);
255 memset (data()[1], eight_bit_uv + 1, lines(1) * stride()[1]);
256 memset (data()[2], eight_bit_uv + 1, lines(2) * stride()[2]);
259 case PIX_FMT_YUV422P9LE:
260 case PIX_FMT_YUV444P9LE:
261 yuv_16_black (nine_bit_uv, false);
264 case PIX_FMT_YUV422P9BE:
265 case PIX_FMT_YUV444P9BE:
266 yuv_16_black (swap_16 (nine_bit_uv), false);
269 case PIX_FMT_YUV422P10LE:
270 case PIX_FMT_YUV444P10LE:
271 yuv_16_black (ten_bit_uv, false);
274 case PIX_FMT_YUV422P16LE:
275 case PIX_FMT_YUV444P16LE:
276 yuv_16_black (sixteen_bit_uv, false);
279 case PIX_FMT_YUV444P10BE:
280 case PIX_FMT_YUV422P10BE:
281 yuv_16_black (swap_16 (ten_bit_uv), false);
284 case AV_PIX_FMT_YUVA420P9BE:
285 case AV_PIX_FMT_YUVA422P9BE:
286 case AV_PIX_FMT_YUVA444P9BE:
287 yuv_16_black (swap_16 (nine_bit_uv), true);
290 case AV_PIX_FMT_YUVA420P9LE:
291 case AV_PIX_FMT_YUVA422P9LE:
292 case AV_PIX_FMT_YUVA444P9LE:
293 yuv_16_black (nine_bit_uv, true);
296 case AV_PIX_FMT_YUVA420P10BE:
297 case AV_PIX_FMT_YUVA422P10BE:
298 case AV_PIX_FMT_YUVA444P10BE:
299 yuv_16_black (swap_16 (ten_bit_uv), true);
302 case AV_PIX_FMT_YUVA420P10LE:
303 case AV_PIX_FMT_YUVA422P10LE:
304 case AV_PIX_FMT_YUVA444P10LE:
305 yuv_16_black (ten_bit_uv, true);
308 case AV_PIX_FMT_YUVA420P16BE:
309 case AV_PIX_FMT_YUVA422P16BE:
310 case AV_PIX_FMT_YUVA444P16BE:
311 yuv_16_black (swap_16 (sixteen_bit_uv), true);
314 case AV_PIX_FMT_YUVA420P16LE:
315 case AV_PIX_FMT_YUVA422P16LE:
316 case AV_PIX_FMT_YUVA444P16LE:
317 yuv_16_black (sixteen_bit_uv, true);
325 memset (data()[0], 0, lines(0) * stride()[0]);
328 case PIX_FMT_UYVY422:
330 int const Y = lines(0);
331 int const X = line_size()[0];
332 uint8_t* p = data()[0];
333 for (int y = 0; y < Y; ++y) {
334 for (int x = 0; x < X / 4; ++x) {
335 *p++ = eight_bit_uv; // Cb
337 *p++ = eight_bit_uv; // Cr
345 throw PixelFormatError ("make_black()", _pixel_format);
350 Image::make_transparent ()
352 if (_pixel_format != PIX_FMT_RGBA) {
353 throw PixelFormatError ("make_transparent()", _pixel_format);
356 memset (data()[0], 0, lines(0) * stride()[0]);
360 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
362 assert (other->pixel_format() == PIX_FMT_RGBA);
363 int const other_bpp = 4;
366 switch (_pixel_format) {
378 int start_tx = position.x;
382 start_ox = -start_tx;
386 int start_ty = position.y;
390 start_oy = -start_ty;
394 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
395 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
396 uint8_t* op = other->data()[0] + oy * other->stride()[0];
397 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
398 float const alpha = float (op[3]) / 255;
399 tp[0] = op[0] + (tp[0] * (1 - alpha));
400 tp[1] = op[1] + (tp[1] * (1 - alpha));
401 tp[2] = op[2] + (tp[2] * (1 - alpha));
402 tp[3] = op[3] + (tp[3] * (1 - alpha));
411 Image::copy (shared_ptr<const Image> other, Position<int> position)
413 /* Only implemented for RGB24 onto RGB24 so far */
414 assert (_pixel_format == PIX_FMT_RGB24 && other->pixel_format() == PIX_FMT_RGB24);
415 assert (position.x >= 0 && position.y >= 0);
417 int const N = min (position.x + other->size().width, size().width) - position.x;
418 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
419 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
420 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
421 memcpy (tp, op, N * 3);
426 Image::read_from_socket (shared_ptr<Socket> socket)
428 for (int i = 0; i < components(); ++i) {
429 uint8_t* p = data()[i];
430 for (int y = 0; y < lines(i); ++y) {
431 socket->read (p, line_size()[i]);
438 Image::write_to_socket (shared_ptr<Socket> socket) const
440 for (int i = 0; i < components(); ++i) {
441 uint8_t* p = data()[i];
442 for (int y = 0; y < lines(i); ++y) {
443 socket->write (p, line_size()[i]);
451 Image::bytes_per_pixel (int c) const
453 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
455 throw PixelFormatError ("lines()", _pixel_format);
458 if (c >= components()) {
462 float bpp[4] = { 0, 0, 0, 0 };
464 bpp[0] = floor ((d->comp[0].depth_minus1 + 1 + 7) / 8);
465 if (d->nb_components > 1) {
466 bpp[1] = floor ((d->comp[1].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
468 if (d->nb_components > 2) {
469 bpp[2] = floor ((d->comp[2].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
471 if (d->nb_components > 3) {
472 bpp[3] = floor ((d->comp[3].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
475 if ((d->flags & PIX_FMT_PLANAR) == 0) {
476 /* Not planar; sum them up */
477 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
483 /** Construct a Image of a given size and format, allocating memory
486 * @param p Pixel format.
487 * @param s Size in pixels.
489 Image::Image (AVPixelFormat p, dcp::Size s, bool aligned)
500 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
501 _data[0] = _data[1] = _data[2] = _data[3] = 0;
503 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
504 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
506 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
507 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
509 for (int i = 0; i < components(); ++i) {
510 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
511 _stride[i] = stride_round_up (i, _line_size, _aligned ? 32 : 1);
513 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
514 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
515 Hence on the last pixel of the last line it reads over the end of
516 the actual data by 1 byte. If the width of an image is a multiple
517 of the stride alignment there will be no padding at the end of image lines.
518 OS X crashes on this illegal read, though other operating systems don't
519 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
520 for that instruction to read safely.
522 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
523 over-reads by more then _avx. I can't follow the code to work out how much,
524 so I'll just over-allocate by 32 bytes and have done with it. Empirical
525 testing suggests that it works.
527 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * lines (i) + 32);
531 Image::Image (Image const & other)
533 , _pixel_format (other._pixel_format)
534 , _aligned (other._aligned)
538 for (int i = 0; i < components(); ++i) {
539 uint8_t* p = _data[i];
540 uint8_t* q = other._data[i];
541 for (int j = 0; j < lines(i); ++j) {
542 memcpy (p, q, _line_size[i]);
544 q += other.stride()[i];
549 Image::Image (AVFrame* frame)
550 : dcp::Image (dcp::Size (frame->width, frame->height))
551 , _pixel_format (static_cast<AVPixelFormat> (frame->format))
556 for (int i = 0; i < components(); ++i) {
557 uint8_t* p = _data[i];
558 uint8_t* q = frame->data[i];
559 for (int j = 0; j < lines(i); ++j) {
560 memcpy (p, q, _line_size[i]);
562 /* AVFrame's linesize is what we call `stride' */
563 q += frame->linesize[i];
568 Image::Image (shared_ptr<const Image> other, bool aligned)
570 , _pixel_format (other->_pixel_format)
575 for (int i = 0; i < components(); ++i) {
576 assert(line_size()[i] == other->line_size()[i]);
577 uint8_t* p = _data[i];
578 uint8_t* q = other->data()[i];
579 for (int j = 0; j < lines(i); ++j) {
580 memcpy (p, q, line_size()[i]);
582 q += other->stride()[i];
588 Image::operator= (Image const & other)
590 if (this == &other) {
600 Image::swap (Image & other)
602 dcp::Image::swap (other);
604 std::swap (_pixel_format, other._pixel_format);
606 for (int i = 0; i < 4; ++i) {
607 std::swap (_data[i], other._data[i]);
608 std::swap (_line_size[i], other._line_size[i]);
609 std::swap (_stride[i], other._stride[i]);
612 std::swap (_aligned, other._aligned);
615 /** Destroy a Image */
618 for (int i = 0; i < components(); ++i) {
623 av_free (_line_size);
634 Image::line_size () const
640 Image::stride () const
652 Image::aligned () const
658 merge (list<PositionImage> images)
660 if (images.empty ()) {
661 return PositionImage ();
664 if (images.size() == 1) {
665 return images.front ();
668 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
669 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
670 all.extend (dcpomatic::Rect<int> (i->position, i->image->size().width, i->image->size().height));
673 shared_ptr<Image> merged (new Image (images.front().image->pixel_format (), dcp::Size (all.width, all.height), true));
674 merged->make_transparent ();
675 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
676 merged->alpha_blend (i->image, i->position - all.position());
679 return PositionImage (merged, all.position ());
683 Image::digest () const
685 MD5Digester digester;
687 for (int i = 0; i < components(); ++i) {
688 digester.add (data()[i], line_size()[i]);
691 return digester.get ();