2 Copyright (C) 2012-2014 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 /** @file src/image.cc
21 * @brief A class to describe a video image.
26 #include <libswscale/swscale.h>
27 #include <libavutil/pixfmt.h>
28 #include <libavutil/pixdesc.h>
31 #include "exceptions.h"
35 #include "md5_digester.h"
44 using boost::shared_ptr;
48 Image::line_factor (int n) const
54 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
56 throw PixelFormatError ("lines()", _pixel_format);
59 return pow (2.0f, d->log2_chroma_h);
62 /** @param n Component index.
63 * @return Number of lines in the image for the given component.
66 Image::lines (int n) const
68 return rint (ceil (static_cast<double>(size().height) / line_factor (n)));
71 /** @return Number of components */
73 Image::components () const
75 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
77 throw PixelFormatError ("components()", _pixel_format);
80 if ((d->flags & PIX_FMT_PLANAR) == 0) {
84 return d->nb_components;
87 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size' */
89 Image::crop_scale_window (Crop crop, dcp::Size inter_size, dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
92 /* Empirical testing suggests that sws_scale() will crash if
93 the input image is not aligned.
97 assert (out_size.width >= inter_size.width);
98 assert (out_size.height >= inter_size.height);
100 /* Here's an image of out_size */
101 shared_ptr<Image> out (new Image (out_format, out_size, out_aligned));
104 /* Size of the image after any crop */
105 dcp::Size const cropped_size = crop.apply (size ());
107 /* Scale context for a scale from cropped_size to inter_size */
108 struct SwsContext* scale_context = sws_getContext (
109 cropped_size.width, cropped_size.height, pixel_format(),
110 inter_size.width, inter_size.height, out_format,
111 scaler->ffmpeg_id (), 0, 0, 0
114 if (!scale_context) {
115 throw StringError (N_("Could not allocate SwsContext"));
118 /* Prepare input data pointers with crop */
119 uint8_t* scale_in_data[components()];
120 for (int c = 0; c < components(); ++c) {
121 scale_in_data[c] = data()[c] + int (rint (bytes_per_pixel(c) * crop.left)) + stride()[c] * (crop.top / line_factor(c));
124 /* Corner of the image within out_size */
125 Position<int> const corner ((out_size.width - inter_size.width) / 2, (out_size.height - inter_size.height) / 2);
127 uint8_t* scale_out_data[out->components()];
128 for (int c = 0; c < out->components(); ++c) {
129 scale_out_data[c] = out->data()[c] + int (rint (out->bytes_per_pixel(c) * corner.x)) + out->stride()[c] * corner.y;
134 scale_in_data, stride(),
135 0, cropped_size.height,
136 scale_out_data, out->stride()
139 sws_freeContext (scale_context);
145 Image::scale (dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
148 /* Empirical testing suggests that sws_scale() will crash if
149 the input image is not aligned.
153 shared_ptr<Image> scaled (new Image (out_format, out_size, out_aligned));
155 struct SwsContext* scale_context = sws_getContext (
156 size().width, size().height, pixel_format(),
157 out_size.width, out_size.height, out_format,
158 scaler->ffmpeg_id (), 0, 0, 0
165 scaled->data(), scaled->stride()
168 sws_freeContext (scale_context);
174 Image::crop (Crop crop, bool aligned) const
176 dcp::Size cropped_size = crop.apply (size ());
177 shared_ptr<Image> out (new Image (pixel_format(), cropped_size, aligned));
179 for (int c = 0; c < components(); ++c) {
180 int const crop_left_in_bytes = bytes_per_pixel(c) * crop.left;
181 /* bytes_per_pixel() could be a fraction; in this case the stride will be rounded
182 up, and we need to make sure that we copy over the width (up to the stride)
183 rather than short of the width; hence the ceil() here.
185 int const cropped_width_in_bytes = ceil (bytes_per_pixel(c) * cropped_size.width);
187 /* Start of the source line, cropped from the top but not the left */
188 uint8_t* in_p = data()[c] + (crop.top / out->line_factor(c)) * stride()[c];
189 uint8_t* out_p = out->data()[c];
191 for (int y = 0; y < out->lines(c); ++y) {
192 memcpy (out_p, in_p + crop_left_in_bytes, cropped_width_in_bytes);
194 out_p += out->stride()[c];
201 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
203 Image::yuv_16_black (uint16_t v, bool alpha)
205 memset (data()[0], 0, lines(0) * stride()[0]);
206 for (int i = 1; i < 3; ++i) {
207 int16_t* p = reinterpret_cast<int16_t*> (data()[i]);
208 for (int y = 0; y < lines(i); ++y) {
209 /* We divide by 2 here because we are writing 2 bytes at a time */
210 for (int x = 0; x < line_size()[i] / 2; ++x) {
213 p += stride()[i] / 2;
218 memset (data()[3], 0, lines(3) * stride()[3]);
223 Image::swap_16 (uint16_t v)
225 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
231 /* U/V black value for 8-bit colour */
232 static uint8_t const eight_bit_uv = (1 << 7) - 1;
233 /* U/V black value for 9-bit colour */
234 static uint16_t const nine_bit_uv = (1 << 8) - 1;
235 /* U/V black value for 10-bit colour */
236 static uint16_t const ten_bit_uv = (1 << 9) - 1;
237 /* U/V black value for 16-bit colour */
238 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
240 switch (_pixel_format) {
241 case PIX_FMT_YUV420P:
242 case PIX_FMT_YUV422P:
243 case PIX_FMT_YUV444P:
244 case PIX_FMT_YUV411P:
245 memset (data()[0], 0, lines(0) * stride()[0]);
246 memset (data()[1], eight_bit_uv, lines(1) * stride()[1]);
247 memset (data()[2], eight_bit_uv, lines(2) * stride()[2]);
250 case PIX_FMT_YUVJ420P:
251 case PIX_FMT_YUVJ422P:
252 case PIX_FMT_YUVJ444P:
253 memset (data()[0], 0, lines(0) * stride()[0]);
254 memset (data()[1], eight_bit_uv + 1, lines(1) * stride()[1]);
255 memset (data()[2], eight_bit_uv + 1, lines(2) * stride()[2]);
258 case PIX_FMT_YUV422P9LE:
259 case PIX_FMT_YUV444P9LE:
260 yuv_16_black (nine_bit_uv, false);
263 case PIX_FMT_YUV422P9BE:
264 case PIX_FMT_YUV444P9BE:
265 yuv_16_black (swap_16 (nine_bit_uv), false);
268 case PIX_FMT_YUV422P10LE:
269 case PIX_FMT_YUV444P10LE:
270 yuv_16_black (ten_bit_uv, false);
273 case PIX_FMT_YUV422P16LE:
274 case PIX_FMT_YUV444P16LE:
275 yuv_16_black (sixteen_bit_uv, false);
278 case PIX_FMT_YUV444P10BE:
279 case PIX_FMT_YUV422P10BE:
280 yuv_16_black (swap_16 (ten_bit_uv), false);
283 case AV_PIX_FMT_YUVA420P9BE:
284 case AV_PIX_FMT_YUVA422P9BE:
285 case AV_PIX_FMT_YUVA444P9BE:
286 yuv_16_black (swap_16 (nine_bit_uv), true);
289 case AV_PIX_FMT_YUVA420P9LE:
290 case AV_PIX_FMT_YUVA422P9LE:
291 case AV_PIX_FMT_YUVA444P9LE:
292 yuv_16_black (nine_bit_uv, true);
295 case AV_PIX_FMT_YUVA420P10BE:
296 case AV_PIX_FMT_YUVA422P10BE:
297 case AV_PIX_FMT_YUVA444P10BE:
298 yuv_16_black (swap_16 (ten_bit_uv), true);
301 case AV_PIX_FMT_YUVA420P10LE:
302 case AV_PIX_FMT_YUVA422P10LE:
303 case AV_PIX_FMT_YUVA444P10LE:
304 yuv_16_black (ten_bit_uv, true);
307 case AV_PIX_FMT_YUVA420P16BE:
308 case AV_PIX_FMT_YUVA422P16BE:
309 case AV_PIX_FMT_YUVA444P16BE:
310 yuv_16_black (swap_16 (sixteen_bit_uv), true);
313 case AV_PIX_FMT_YUVA420P16LE:
314 case AV_PIX_FMT_YUVA422P16LE:
315 case AV_PIX_FMT_YUVA444P16LE:
316 yuv_16_black (sixteen_bit_uv, true);
324 case PIX_FMT_RGB555LE:
325 case PIX_FMT_RGB48LE:
326 case PIX_FMT_RGB48BE:
327 memset (data()[0], 0, lines(0) * stride()[0]);
330 case PIX_FMT_UYVY422:
332 int const Y = lines(0);
333 int const X = line_size()[0];
334 uint8_t* p = data()[0];
335 for (int y = 0; y < Y; ++y) {
336 for (int x = 0; x < X / 4; ++x) {
337 *p++ = eight_bit_uv; // Cb
339 *p++ = eight_bit_uv; // Cr
347 throw PixelFormatError ("make_black()", _pixel_format);
352 Image::make_transparent ()
354 if (_pixel_format != PIX_FMT_RGBA) {
355 throw PixelFormatError ("make_transparent()", _pixel_format);
358 memset (data()[0], 0, lines(0) * stride()[0]);
362 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
364 assert (other->pixel_format() == PIX_FMT_RGBA);
365 int const other_bpp = 4;
368 switch (_pixel_format) {
380 int start_tx = position.x;
384 start_ox = -start_tx;
388 int start_ty = position.y;
392 start_oy = -start_ty;
396 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
397 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
398 uint8_t* op = other->data()[0] + oy * other->stride()[0];
399 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
400 float const alpha = float (op[3]) / 255;
401 tp[0] = op[0] + (tp[0] * (1 - alpha));
402 tp[1] = op[1] + (tp[1] * (1 - alpha));
403 tp[2] = op[2] + (tp[2] * (1 - alpha));
404 tp[3] = op[3] + (tp[3] * (1 - alpha));
413 Image::copy (shared_ptr<const Image> other, Position<int> position)
415 /* Only implemented for RGB24 onto RGB24 so far */
416 assert (_pixel_format == PIX_FMT_RGB24 && other->pixel_format() == PIX_FMT_RGB24);
417 assert (position.x >= 0 && position.y >= 0);
419 int const N = min (position.x + other->size().width, size().width) - position.x;
420 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
421 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
422 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
423 memcpy (tp, op, N * 3);
428 Image::read_from_socket (shared_ptr<Socket> socket)
430 for (int i = 0; i < components(); ++i) {
431 uint8_t* p = data()[i];
432 for (int y = 0; y < lines(i); ++y) {
433 socket->read (p, line_size()[i]);
440 Image::write_to_socket (shared_ptr<Socket> socket) const
442 for (int i = 0; i < components(); ++i) {
443 uint8_t* p = data()[i];
444 for (int y = 0; y < lines(i); ++y) {
445 socket->write (p, line_size()[i]);
453 Image::bytes_per_pixel (int c) const
455 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
457 throw PixelFormatError ("lines()", _pixel_format);
460 if (c >= components()) {
464 float bpp[4] = { 0, 0, 0, 0 };
466 bpp[0] = floor ((d->comp[0].depth_minus1 + 1 + 7) / 8);
467 if (d->nb_components > 1) {
468 bpp[1] = floor ((d->comp[1].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
470 if (d->nb_components > 2) {
471 bpp[2] = floor ((d->comp[2].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
473 if (d->nb_components > 3) {
474 bpp[3] = floor ((d->comp[3].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
477 if ((d->flags & PIX_FMT_PLANAR) == 0) {
478 /* Not planar; sum them up */
479 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
485 /** Construct a Image of a given size and format, allocating memory
488 * @param p Pixel format.
489 * @param s Size in pixels.
491 Image::Image (AVPixelFormat p, dcp::Size s, bool aligned)
502 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
503 _data[0] = _data[1] = _data[2] = _data[3] = 0;
505 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
506 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
508 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
509 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
511 for (int i = 0; i < components(); ++i) {
512 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
513 _stride[i] = stride_round_up (i, _line_size, _aligned ? 32 : 1);
515 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
516 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
517 Hence on the last pixel of the last line it reads over the end of
518 the actual data by 1 byte. If the width of an image is a multiple
519 of the stride alignment there will be no padding at the end of image lines.
520 OS X crashes on this illegal read, though other operating systems don't
521 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
522 for that instruction to read safely.
524 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
525 over-reads by more then _avx. I can't follow the code to work out how much,
526 so I'll just over-allocate by 32 bytes and have done with it. Empirical
527 testing suggests that it works.
529 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * lines (i) + 32);
533 Image::Image (Image const & other)
535 , _pixel_format (other._pixel_format)
536 , _aligned (other._aligned)
540 for (int i = 0; i < components(); ++i) {
541 uint8_t* p = _data[i];
542 uint8_t* q = other._data[i];
543 for (int j = 0; j < lines(i); ++j) {
544 memcpy (p, q, _line_size[i]);
546 q += other.stride()[i];
551 Image::Image (AVFrame* frame)
552 : dcp::Image (dcp::Size (frame->width, frame->height))
553 , _pixel_format (static_cast<AVPixelFormat> (frame->format))
558 for (int i = 0; i < components(); ++i) {
559 uint8_t* p = _data[i];
560 uint8_t* q = frame->data[i];
561 for (int j = 0; j < lines(i); ++j) {
562 memcpy (p, q, _line_size[i]);
564 /* AVFrame's linesize is what we call `stride' */
565 q += frame->linesize[i];
570 Image::Image (shared_ptr<const Image> other, bool aligned)
572 , _pixel_format (other->_pixel_format)
577 for (int i = 0; i < components(); ++i) {
578 assert(line_size()[i] == other->line_size()[i]);
579 uint8_t* p = _data[i];
580 uint8_t* q = other->data()[i];
581 for (int j = 0; j < lines(i); ++j) {
582 memcpy (p, q, line_size()[i]);
584 q += other->stride()[i];
590 Image::operator= (Image const & other)
592 if (this == &other) {
602 Image::swap (Image & other)
604 dcp::Image::swap (other);
606 std::swap (_pixel_format, other._pixel_format);
608 for (int i = 0; i < 4; ++i) {
609 std::swap (_data[i], other._data[i]);
610 std::swap (_line_size[i], other._line_size[i]);
611 std::swap (_stride[i], other._stride[i]);
614 std::swap (_aligned, other._aligned);
617 /** Destroy a Image */
620 for (int i = 0; i < components(); ++i) {
625 av_free (_line_size);
636 Image::line_size () const
642 Image::stride () const
654 Image::aligned () const
660 merge (list<PositionImage> images)
662 if (images.empty ()) {
663 return PositionImage ();
666 if (images.size() == 1) {
667 return images.front ();
670 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
671 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
672 all.extend (dcpomatic::Rect<int> (i->position, i->image->size().width, i->image->size().height));
675 shared_ptr<Image> merged (new Image (images.front().image->pixel_format (), dcp::Size (all.width, all.height), true));
676 merged->make_transparent ();
677 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
678 merged->alpha_blend (i->image, i->position - all.position());
681 return PositionImage (merged, all.position ());
685 Image::digest () const
687 MD5Digester digester;
689 for (int i = 0; i < components(); ++i) {
690 digester.add (data()[i], line_size()[i]);
693 return digester.get ();
697 operator== (Image const & a, Image const & b)
699 if (a.components() != b.components() || a.pixel_format() != b.pixel_format() || a.aligned() != b.aligned()) {
703 for (int c = 0; c < a.components(); ++c) {
704 if (a.lines(c) != b.lines(c) || a.line_size()[c] != b.line_size()[c] || a.stride()[c] != b.stride()[c]) {
708 uint8_t* p = a.data()[c];
709 uint8_t* q = b.data()[c];
710 for (int y = 0; y < a.lines(c); ++y) {
711 if (memcmp (p, q, a.line_size()[c]) != 0) {
724 Image::fade (float f)
726 switch (_pixel_format) {
727 case PIX_FMT_YUV420P:
728 case PIX_FMT_YUV422P:
729 case PIX_FMT_YUV444P:
730 case PIX_FMT_YUV411P:
731 case PIX_FMT_YUVJ420P:
732 case PIX_FMT_YUVJ422P:
733 case PIX_FMT_YUVJ444P:
739 case PIX_FMT_RGB555LE:
741 for (int c = 0; c < 3; ++c) {
742 uint8_t* p = data()[c];
743 for (int y = 0; y < lines(c); ++y) {
745 for (int x = 0; x < line_size()[c]; ++x) {
746 *q = int (float (*q) * f);
754 case PIX_FMT_YUV422P9LE:
755 case PIX_FMT_YUV444P9LE:
756 case PIX_FMT_YUV422P10LE:
757 case PIX_FMT_YUV444P10LE:
758 case PIX_FMT_YUV422P16LE:
759 case PIX_FMT_YUV444P16LE:
760 case AV_PIX_FMT_YUVA420P9LE:
761 case AV_PIX_FMT_YUVA422P9LE:
762 case AV_PIX_FMT_YUVA444P9LE:
763 case AV_PIX_FMT_YUVA420P10LE:
764 case AV_PIX_FMT_YUVA422P10LE:
765 case AV_PIX_FMT_YUVA444P10LE:
766 /* 16-bit little-endian */
767 for (int c = 0; c < 3; ++c) {
768 int const stride_pixels = stride()[c] / 2;
769 int const line_size_pixels = line_size()[c] / 2;
770 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
771 for (int y = 0; y < lines(c); ++y) {
773 for (int x = 0; x < line_size_pixels; ++x) {
774 *q = int (float (*q) * f);
782 case PIX_FMT_YUV422P9BE:
783 case PIX_FMT_YUV444P9BE:
784 case PIX_FMT_YUV444P10BE:
785 case PIX_FMT_YUV422P10BE:
786 case AV_PIX_FMT_YUVA420P9BE:
787 case AV_PIX_FMT_YUVA422P9BE:
788 case AV_PIX_FMT_YUVA444P9BE:
789 case AV_PIX_FMT_YUVA420P10BE:
790 case AV_PIX_FMT_YUVA422P10BE:
791 case AV_PIX_FMT_YUVA444P10BE:
792 case AV_PIX_FMT_YUVA420P16BE:
793 case AV_PIX_FMT_YUVA422P16BE:
794 case AV_PIX_FMT_YUVA444P16BE:
795 /* 16-bit big-endian */
796 for (int c = 0; c < 3; ++c) {
797 int const stride_pixels = stride()[c] / 2;
798 int const line_size_pixels = line_size()[c] / 2;
799 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
800 for (int y = 0; y < lines(c); ++y) {
802 for (int x = 0; x < line_size_pixels; ++x) {
803 *q = swap_16 (int (float (swap_16 (*q)) * f));
811 case PIX_FMT_UYVY422:
813 int const Y = lines(0);
814 int const X = line_size()[0];
815 uint8_t* p = data()[0];
816 for (int y = 0; y < Y; ++y) {
817 for (int x = 0; x < X; ++x) {
818 *p = int (float (*p) * f);
826 throw PixelFormatError ("fade()", _pixel_format);