Add overflow checks for opj_aligned_malloc (#841)

[openjpeg.git] / src / lib / openjp2 / dwt.c

diff --git a/src/lib/openjp2/dwt.c b/src/lib/openjp2/dwt.c
index 2d793bb6a15abf8f1484f8b1f72a3cdbd7b8c977..98ead59e2e9ce3723f3c76e7224fbefc2a950405 100644 (file)
--- a/src/lib/openjp2/dwt.c
+++ b/src/lib/openjp2/dwt.c
@@ -395,7 +395,7 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
 
        OPJ_INT32 rw;                   /* width of the resolution level computed   */
        OPJ_INT32 rh;                   /* height of the resolution level computed  */
-       OPJ_UINT32 l_data_size;
+       size_t l_data_size;
 
        opj_tcd_resolution_t * l_cur_res = 0;
        opj_tcd_resolution_t * l_last_res = 0;
@@ -407,8 +407,14 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
        l_cur_res = tilec->resolutions + l;
        l_last_res = l_cur_res - 1;
 
-       l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions) * (OPJ_UINT32)sizeof(OPJ_INT32);
-       bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size);
+       l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions);
+       /* overflow check */
+       if (l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       l_data_size *= sizeof(OPJ_INT32);
+       bj = (OPJ_INT32*)opj_malloc(l_data_size);
        /* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */
        /* in that case, so do not error out */
        if (l_data_size != 0 && ! bj) {
@@ -638,7 +644,13 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_thread_pool_t* tp, opj_tcd_tilecomp_t* t
                return OPJ_TRUE;
        }
        num_threads = opj_thread_pool_get_thread_count(tp);
-       h_mem_size = opj_dwt_max_resolution(tr, numres) * sizeof(OPJ_INT32);
+       h_mem_size = opj_dwt_max_resolution(tr, numres);
+       /* overflow check */
+       if (h_mem_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       h_mem_size *= sizeof(OPJ_INT32);
        h.mem = (OPJ_INT32*)opj_aligned_malloc(h_mem_size);
        if (! h.mem){
                /* FIXME event manager error callback */
@@ -1003,7 +1015,21 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32
 
        OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);
 
-       h.wavelet = (opj_v4_t*) opj_aligned_malloc((opj_dwt_max_resolution(res, numres)+5) * sizeof(opj_v4_t));
+       size_t l_data_size;
+       
+       l_data_size = opj_dwt_max_resolution(res, numres);
+       /* overflow check */
+       if (l_data_size > (SIZE_MAX - 5U)) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       l_data_size += 5U;
+       /* overflow check */
+       if (l_data_size > (SIZE_MAX / sizeof(opj_v4_t))) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       h.wavelet = (opj_v4_t*) opj_aligned_malloc(l_data_size * sizeof(opj_v4_t));
        if (!h.wavelet) {
                /* FIXME event manager error callback */
                return OPJ_FALSE;