2 Copyright (C) 2012-2014 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 /** @file src/image.cc
21 * @brief A class to describe a video image.
26 #include <libswscale/swscale.h>
27 #include <libavutil/pixfmt.h>
28 #include <libavutil/pixdesc.h>
31 #include "exceptions.h"
35 #include "md5_digester.h"
44 using std::stringstream;
45 using boost::shared_ptr;
49 Image::line_factor (int n) const
55 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
57 throw PixelFormatError ("lines()", _pixel_format);
60 return pow (2.0f, d->log2_chroma_h);
63 /** @param n Component index.
64 * @return Number of lines in the image for the given component.
67 Image::lines (int n) const
69 return rint (ceil (static_cast<double>(size().height) / line_factor (n)));
72 /** @return Number of components */
74 Image::components () const
76 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
78 throw PixelFormatError ("components()", _pixel_format);
81 if ((d->flags & PIX_FMT_PLANAR) == 0) {
85 return d->nb_components;
88 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size' */
90 Image::crop_scale_window (Crop crop, dcp::Size inter_size, dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
93 /* Empirical testing suggests that sws_scale() will crash if
94 the input image is not aligned.
98 assert (out_size.width >= inter_size.width);
99 assert (out_size.height >= inter_size.height);
101 /* Here's an image of out_size */
102 shared_ptr<Image> out (new Image (out_format, out_size, out_aligned));
105 /* Size of the image after any crop */
106 dcp::Size const cropped_size = crop.apply (size ());
108 /* Scale context for a scale from cropped_size to inter_size */
109 struct SwsContext* scale_context = sws_getContext (
110 cropped_size.width, cropped_size.height, pixel_format(),
111 inter_size.width, inter_size.height, out_format,
112 scaler->ffmpeg_id (), 0, 0, 0
115 if (!scale_context) {
116 throw StringError (N_("Could not allocate SwsContext"));
119 /* Prepare input data pointers with crop */
120 uint8_t* scale_in_data[components()];
121 for (int c = 0; c < components(); ++c) {
122 scale_in_data[c] = data()[c] + int (rint (bytes_per_pixel(c) * crop.left)) + stride()[c] * (crop.top / line_factor(c));
125 /* Corner of the image within out_size */
126 Position<int> const corner ((out_size.width - inter_size.width) / 2, (out_size.height - inter_size.height) / 2);
128 uint8_t* scale_out_data[out->components()];
129 for (int c = 0; c < out->components(); ++c) {
130 scale_out_data[c] = out->data()[c] + int (rint (out->bytes_per_pixel(c) * corner.x)) + out->stride()[c] * corner.y;
135 scale_in_data, stride(),
136 0, cropped_size.height,
137 scale_out_data, out->stride()
140 sws_freeContext (scale_context);
146 Image::scale (dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
149 /* Empirical testing suggests that sws_scale() will crash if
150 the input image is not aligned.
154 shared_ptr<Image> scaled (new Image (out_format, out_size, out_aligned));
156 struct SwsContext* scale_context = sws_getContext (
157 size().width, size().height, pixel_format(),
158 out_size.width, out_size.height, out_format,
159 scaler->ffmpeg_id (), 0, 0, 0
166 scaled->data(), scaled->stride()
169 sws_freeContext (scale_context);
175 Image::crop (Crop crop, bool aligned) const
177 dcp::Size cropped_size = crop.apply (size ());
178 shared_ptr<Image> out (new Image (pixel_format(), cropped_size, aligned));
180 for (int c = 0; c < components(); ++c) {
181 int const crop_left_in_bytes = bytes_per_pixel(c) * crop.left;
182 /* bytes_per_pixel() could be a fraction; in this case the stride will be rounded
183 up, and we need to make sure that we copy over the width (up to the stride)
184 rather than short of the width; hence the ceil() here.
186 int const cropped_width_in_bytes = ceil (bytes_per_pixel(c) * cropped_size.width);
188 /* Start of the source line, cropped from the top but not the left */
189 uint8_t* in_p = data()[c] + (crop.top / out->line_factor(c)) * stride()[c];
190 uint8_t* out_p = out->data()[c];
192 for (int y = 0; y < out->lines(c); ++y) {
193 memcpy (out_p, in_p + crop_left_in_bytes, cropped_width_in_bytes);
195 out_p += out->stride()[c];
202 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
204 Image::yuv_16_black (uint16_t v, bool alpha)
206 memset (data()[0], 0, lines(0) * stride()[0]);
207 for (int i = 1; i < 3; ++i) {
208 int16_t* p = reinterpret_cast<int16_t*> (data()[i]);
209 for (int y = 0; y < lines(i); ++y) {
210 /* We divide by 2 here because we are writing 2 bytes at a time */
211 for (int x = 0; x < line_size()[i] / 2; ++x) {
214 p += stride()[i] / 2;
219 memset (data()[3], 0, lines(3) * stride()[3]);
224 Image::swap_16 (uint16_t v)
226 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
232 /* U/V black value for 8-bit colour */
233 static uint8_t const eight_bit_uv = (1 << 7) - 1;
234 /* U/V black value for 9-bit colour */
235 static uint16_t const nine_bit_uv = (1 << 8) - 1;
236 /* U/V black value for 10-bit colour */
237 static uint16_t const ten_bit_uv = (1 << 9) - 1;
238 /* U/V black value for 16-bit colour */
239 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
241 switch (_pixel_format) {
242 case PIX_FMT_YUV420P:
243 case PIX_FMT_YUV422P:
244 case PIX_FMT_YUV444P:
245 case PIX_FMT_YUV411P:
246 memset (data()[0], 0, lines(0) * stride()[0]);
247 memset (data()[1], eight_bit_uv, lines(1) * stride()[1]);
248 memset (data()[2], eight_bit_uv, lines(2) * stride()[2]);
251 case PIX_FMT_YUVJ420P:
252 case PIX_FMT_YUVJ422P:
253 case PIX_FMT_YUVJ444P:
254 memset (data()[0], 0, lines(0) * stride()[0]);
255 memset (data()[1], eight_bit_uv + 1, lines(1) * stride()[1]);
256 memset (data()[2], eight_bit_uv + 1, lines(2) * stride()[2]);
259 case PIX_FMT_YUV422P9LE:
260 case PIX_FMT_YUV444P9LE:
261 yuv_16_black (nine_bit_uv, false);
264 case PIX_FMT_YUV422P9BE:
265 case PIX_FMT_YUV444P9BE:
266 yuv_16_black (swap_16 (nine_bit_uv), false);
269 case PIX_FMT_YUV422P10LE:
270 case PIX_FMT_YUV444P10LE:
271 yuv_16_black (ten_bit_uv, false);
274 case PIX_FMT_YUV422P16LE:
275 case PIX_FMT_YUV444P16LE:
276 yuv_16_black (sixteen_bit_uv, false);
279 case PIX_FMT_YUV444P10BE:
280 case PIX_FMT_YUV422P10BE:
281 yuv_16_black (swap_16 (ten_bit_uv), false);
284 case AV_PIX_FMT_YUVA420P9BE:
285 case AV_PIX_FMT_YUVA422P9BE:
286 case AV_PIX_FMT_YUVA444P9BE:
287 yuv_16_black (swap_16 (nine_bit_uv), true);
290 case AV_PIX_FMT_YUVA420P9LE:
291 case AV_PIX_FMT_YUVA422P9LE:
292 case AV_PIX_FMT_YUVA444P9LE:
293 yuv_16_black (nine_bit_uv, true);
296 case AV_PIX_FMT_YUVA420P10BE:
297 case AV_PIX_FMT_YUVA422P10BE:
298 case AV_PIX_FMT_YUVA444P10BE:
299 yuv_16_black (swap_16 (ten_bit_uv), true);
302 case AV_PIX_FMT_YUVA420P10LE:
303 case AV_PIX_FMT_YUVA422P10LE:
304 case AV_PIX_FMT_YUVA444P10LE:
305 yuv_16_black (ten_bit_uv, true);
308 case AV_PIX_FMT_YUVA420P16BE:
309 case AV_PIX_FMT_YUVA422P16BE:
310 case AV_PIX_FMT_YUVA444P16BE:
311 yuv_16_black (swap_16 (sixteen_bit_uv), true);
314 case AV_PIX_FMT_YUVA420P16LE:
315 case AV_PIX_FMT_YUVA422P16LE:
316 case AV_PIX_FMT_YUVA444P16LE:
317 yuv_16_black (sixteen_bit_uv, true);
325 case PIX_FMT_RGB555LE:
326 memset (data()[0], 0, lines(0) * stride()[0]);
329 case PIX_FMT_UYVY422:
331 int const Y = lines(0);
332 int const X = line_size()[0];
333 uint8_t* p = data()[0];
334 for (int y = 0; y < Y; ++y) {
335 for (int x = 0; x < X / 4; ++x) {
336 *p++ = eight_bit_uv; // Cb
338 *p++ = eight_bit_uv; // Cr
346 throw PixelFormatError ("make_black()", _pixel_format);
351 Image::make_transparent ()
353 if (_pixel_format != PIX_FMT_RGBA) {
354 throw PixelFormatError ("make_transparent()", _pixel_format);
357 memset (data()[0], 0, lines(0) * stride()[0]);
361 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
363 assert (other->pixel_format() == PIX_FMT_RGBA);
364 int const other_bpp = 4;
367 switch (_pixel_format) {
379 int start_tx = position.x;
383 start_ox = -start_tx;
387 int start_ty = position.y;
391 start_oy = -start_ty;
395 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
396 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
397 uint8_t* op = other->data()[0] + oy * other->stride()[0];
398 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
399 float const alpha = float (op[3]) / 255;
400 tp[0] = op[0] + (tp[0] * (1 - alpha));
401 tp[1] = op[1] + (tp[1] * (1 - alpha));
402 tp[2] = op[2] + (tp[2] * (1 - alpha));
403 tp[3] = op[3] + (tp[3] * (1 - alpha));
412 Image::copy (shared_ptr<const Image> other, Position<int> position)
414 /* Only implemented for RGB24 onto RGB24 so far */
415 assert (_pixel_format == PIX_FMT_RGB24 && other->pixel_format() == PIX_FMT_RGB24);
416 assert (position.x >= 0 && position.y >= 0);
418 int const N = min (position.x + other->size().width, size().width) - position.x;
419 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
420 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
421 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
422 memcpy (tp, op, N * 3);
427 Image::read_from_socket (shared_ptr<Socket> socket)
429 for (int i = 0; i < components(); ++i) {
430 uint8_t* p = data()[i];
431 for (int y = 0; y < lines(i); ++y) {
432 socket->read (p, line_size()[i]);
439 Image::write_to_socket (shared_ptr<Socket> socket) const
441 for (int i = 0; i < components(); ++i) {
442 uint8_t* p = data()[i];
443 for (int y = 0; y < lines(i); ++y) {
444 socket->write (p, line_size()[i]);
452 Image::bytes_per_pixel (int c) const
454 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
456 throw PixelFormatError ("lines()", _pixel_format);
459 if (c >= components()) {
463 float bpp[4] = { 0, 0, 0, 0 };
465 bpp[0] = floor ((d->comp[0].depth_minus1 + 1 + 7) / 8);
466 if (d->nb_components > 1) {
467 bpp[1] = floor ((d->comp[1].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
469 if (d->nb_components > 2) {
470 bpp[2] = floor ((d->comp[2].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
472 if (d->nb_components > 3) {
473 bpp[3] = floor ((d->comp[3].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
476 if ((d->flags & PIX_FMT_PLANAR) == 0) {
477 /* Not planar; sum them up */
478 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
484 /** Construct a Image of a given size and format, allocating memory
487 * @param p Pixel format.
488 * @param s Size in pixels.
490 Image::Image (AVPixelFormat p, dcp::Size s, bool aligned)
501 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
502 _data[0] = _data[1] = _data[2] = _data[3] = 0;
504 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
505 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
507 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
508 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
510 for (int i = 0; i < components(); ++i) {
511 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
512 _stride[i] = stride_round_up (i, _line_size, _aligned ? 32 : 1);
514 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
515 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
516 Hence on the last pixel of the last line it reads over the end of
517 the actual data by 1 byte. If the width of an image is a multiple
518 of the stride alignment there will be no padding at the end of image lines.
519 OS X crashes on this illegal read, though other operating systems don't
520 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
521 for that instruction to read safely.
523 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
524 over-reads by more then _avx. I can't follow the code to work out how much,
525 so I'll just over-allocate by 32 bytes and have done with it. Empirical
526 testing suggests that it works.
528 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * lines (i) + 32);
532 Image::Image (Image const & other)
534 , _pixel_format (other._pixel_format)
535 , _aligned (other._aligned)
539 for (int i = 0; i < components(); ++i) {
540 uint8_t* p = _data[i];
541 uint8_t* q = other._data[i];
542 for (int j = 0; j < lines(i); ++j) {
543 memcpy (p, q, _line_size[i]);
545 q += other.stride()[i];
550 Image::Image (AVFrame* frame)
551 : dcp::Image (dcp::Size (frame->width, frame->height))
552 , _pixel_format (static_cast<AVPixelFormat> (frame->format))
557 for (int i = 0; i < components(); ++i) {
558 uint8_t* p = _data[i];
559 uint8_t* q = frame->data[i];
560 for (int j = 0; j < lines(i); ++j) {
561 memcpy (p, q, _line_size[i]);
563 /* AVFrame's linesize is what we call `stride' */
564 q += frame->linesize[i];
569 Image::Image (shared_ptr<const Image> other, bool aligned)
571 , _pixel_format (other->_pixel_format)
576 for (int i = 0; i < components(); ++i) {
577 assert(line_size()[i] == other->line_size()[i]);
578 uint8_t* p = _data[i];
579 uint8_t* q = other->data()[i];
580 for (int j = 0; j < lines(i); ++j) {
581 memcpy (p, q, line_size()[i]);
583 q += other->stride()[i];
589 Image::operator= (Image const & other)
591 if (this == &other) {
601 Image::swap (Image & other)
603 dcp::Image::swap (other);
605 std::swap (_pixel_format, other._pixel_format);
607 for (int i = 0; i < 4; ++i) {
608 std::swap (_data[i], other._data[i]);
609 std::swap (_line_size[i], other._line_size[i]);
610 std::swap (_stride[i], other._stride[i]);
613 std::swap (_aligned, other._aligned);
616 /** Destroy a Image */
619 for (int i = 0; i < components(); ++i) {
624 av_free (_line_size);
635 Image::line_size () const
641 Image::stride () const
653 Image::aligned () const
659 merge (list<PositionImage> images)
661 if (images.empty ()) {
662 return PositionImage ();
665 if (images.size() == 1) {
666 return images.front ();
669 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
670 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
671 all.extend (dcpomatic::Rect<int> (i->position, i->image->size().width, i->image->size().height));
674 shared_ptr<Image> merged (new Image (images.front().image->pixel_format (), dcp::Size (all.width, all.height), true));
675 merged->make_transparent ();
676 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
677 merged->alpha_blend (i->image, i->position - all.position());
680 return PositionImage (merged, all.position ());
684 Image::digest () const
686 MD5Digester digester;
688 for (int i = 0; i < components(); ++i) {
689 digester.add (data()[i], line_size()[i]);
692 return digester.get ();