Add overflow checks for opj_aligned_malloc

See
https://pdfium.googlesource.com/pdfium/+/b20ab6c7acb3be1393461eb650ca8fa
4660c937e/third_party/libopenjpeg20/0020-opj_aligned_malloc.patch

diff --git a/src/lib/openjp2/dwt.c b/src/lib/openjp2/dwt.c
index 4fce8b209c085dcf90742cdb675825376e5c95de..b39f63e7d3714957b06d5fd961dc01c78b78ba43 100644 (file)
--- a/src/lib/openjp2/dwt.c
+++ b/src/lib/openjp2/dwt.c
@@ -407,11 +407,19 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
        l_cur_res = tilec->resolutions + l;
        l_last_res = l_cur_res - 1;
 
-       l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions) * (OPJ_UINT32)sizeof(OPJ_INT32);
-       bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size);
+       l_data_size = opj_dwt_max_resolution(tilec->resolutions, tilec->numresolutions);
+ 
+       /* overflow check */
+       if ((size_t)l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+
+       bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size * sizeof(OPJ_INT32));
        /* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */
        /* in that case, so do not error out */
        if (l_data_size != 0 && ! bj) {
+               /* FIXME event manager error callback */
                return OPJ_FALSE;
        }
        i = l;
@@ -569,11 +577,22 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_tcd_tilecomp_t* tilec, OPJ_UINT32 numres
        OPJ_UINT32 rh = (OPJ_UINT32)(tr->y1 - tr->y0);  /* height of the resolution level computed */
 
        OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);
+
+       OPJ_UINT32 mr; /* max resolution */
        
        if (numres == 1U) {
                return OPJ_TRUE;
        }
-       h.mem = (OPJ_INT32*)opj_aligned_malloc(opj_dwt_max_resolution(tr, numres) * sizeof(OPJ_INT32));
+       
+       mr = opj_dwt_max_resolution(tr, numres);
+       
+       /* overflow check */
+       if ((size_t)mr > (SIZE_MAX / sizeof(OPJ_INT32))) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       
+       h.mem = (OPJ_INT32*)opj_aligned_malloc((size_t)mr * sizeof(OPJ_INT32));
        if (! h.mem){
                /* FIXME event manager error callback */
                return OPJ_FALSE;
@@ -845,8 +864,24 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32
        OPJ_UINT32 rh = (OPJ_UINT32)(res->y1 - res->y0);        /* height of the resolution level computed */
 
        OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);
+       
+       OPJ_UINT32 mr; /* max resolution */
+       
+       mr = opj_dwt_max_resolution(res, numres);
+       
+       /* overflow check */
+       if (mr > (0xFFFFFFFFU /* UINT32_MAX */ - 5U)) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       mr += 5U;
+       
+       if ((size_t)mr > (SIZE_MAX / sizeof(opj_v4_t))) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
 
-       h.wavelet = (opj_v4_t*) opj_aligned_malloc((opj_dwt_max_resolution(res, numres)+5) * sizeof(opj_v4_t));
+       h.wavelet = (opj_v4_t*) opj_aligned_malloc((size_t)mr * sizeof(opj_v4_t));
        if (!h.wavelet) {
                /* FIXME event manager error callback */
                return OPJ_FALSE;

diff --git a/src/lib/openjp2/pi.c b/src/lib/openjp2/pi.c
index 809b33d731dca0b4df4868949d29a5a474cc2a3a..41a2f046fde9d284ed813d06d14c4a44df9fa345 100644 (file)
--- a/src/lib/openjp2/pi.c
+++ b/src/lib/openjp2/pi.c
@@ -1238,14 +1238,14 @@ opj_pi_iterator_t *opj_pi_create_decode(opj_image_t *p_image,
 
        /* memory allocation for include */
        /* prevent an integer overflow issue */
+       /* 0 < l_tcp->numlayers < 65536 c.f. opj_j2k_read_cod in j2k.c */
        l_current_pi->include = 00;
        if (l_step_l <= (SIZE_MAX / (l_tcp->numlayers + 1U)))
        {
                l_current_pi->include = (OPJ_INT16*) opj_calloc((size_t)(l_tcp->numlayers + 1U) * l_step_l, sizeof(OPJ_INT16));
        }
 
-       if
-               (!l_current_pi->include)
+       if (!l_current_pi->include)
        {
                opj_free(l_tmp_data);
                opj_free(l_tmp_ptr);

diff --git a/src/lib/openjp2/t1.c b/src/lib/openjp2/t1.c
index cb5a1cefd30d460f4d307bea686d39e64a0ea105..211708fcff65e20fe5cbd31dc690489584863b6f 100644 (file)
--- a/src/lib/openjp2/t1.c
+++ b/src/lib/openjp2/t1.c
@@ -1166,41 +1166,71 @@ static OPJ_BOOL opj_t1_allocate_buffers(
                OPJ_UINT32 w,
                OPJ_UINT32 h)
 {
-       OPJ_UINT32 datasize=w * h;
        OPJ_UINT32 flagssize;
 
        /* encoder uses tile buffer, so no need to allocate */
        if (!t1->encoder) {
+               OPJ_UINT32 datasize;
+               
+               /* Overflow check */
+               if ((w > 0U) && (h > (0xFFFFFFFFU /* UINT32_MAX */ / w))) {
+                       /* FIXME event manager error callback */
+                       return OPJ_FALSE;
+               }
+               datasize = w * h;
+               
+               /* Overflow check */
+               if ((size_t)datasize > (SIZE_MAX / sizeof(OPJ_INT32))) {
+                       /* FIXME event manager error callback */
+                       return OPJ_FALSE;
+               }
+               
                if(datasize > t1->datasize){
                        opj_aligned_free(t1->data);
-                       t1->data = (OPJ_INT32*) opj_aligned_malloc(datasize * sizeof(OPJ_INT32));
+                       t1->data = (OPJ_INT32*) opj_aligned_malloc((size_t)datasize * sizeof(OPJ_INT32));
                        if(!t1->data){
                                /* FIXME event manager error callback */
                                return OPJ_FALSE;
                        }
-                       t1->datasize=datasize;
+                       t1->datasize = datasize;
                }
                /* memset first arg is declared to never be null by gcc */
                if (t1->data != NULL) {
-                       memset(t1->data,0,datasize * sizeof(OPJ_INT32));
+                       memset(t1->data, 0, (size_t)datasize * sizeof(OPJ_INT32));
                }
        }
-       t1->flags_stride=w+2;
-       flagssize=t1->flags_stride * (h+2);
+       
+       if ((w > (0xFFFFFFFFU /* UINT32_MAX */ - 2U)) || (h > (0xFFFFFFFFU /* UINT32_MAX */ - 2U))) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       
+       t1->flags_stride = w + 2U; /* can't be 0U */
+       if ((h + 2U) > (0xFFFFFFFFU /* UINT32_MAX */ / t1->flags_stride)) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       flagssize = t1->flags_stride * (h + 2U);
 
+       /* Overflow check */
+       if ((size_t)flagssize > (SIZE_MAX / sizeof(opj_flag_t))) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       
        if(flagssize > t1->flagssize){
                opj_aligned_free(t1->flags);
-               t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t));
+               t1->flags = (opj_flag_t*) opj_aligned_malloc((size_t)flagssize * sizeof(opj_flag_t));
                if(!t1->flags){
                        /* FIXME event manager error callback */
                        return OPJ_FALSE;
                }
-               t1->flagssize=flagssize;
+               t1->flagssize = flagssize;
        }
-       memset(t1->flags,0,flagssize * sizeof(opj_flag_t));
+       memset(t1->flags, 0, (size_t)flagssize * sizeof(opj_flag_t));
 
-       t1->w=w;
-       t1->h=h;
+       t1->w = w;
+       t1->h = h;
 
        return OPJ_TRUE;
 }