[1.5] This commit hides symptoms of CVE-2009-5030
[openjpeg.git] / libopenjpeg / tcd.c
index 4a58ebeb71407cbf5684bfb7b26c9a831f6a8e41..839ca7c3a919600918b591d7b4908d64c435717d 100644 (file)
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#define _ISOC99_SOURCE /* lrintf is C99 */
 #include "opj_includes.h"
 
 void tcd_dump(FILE *fd, opj_tcd_t *tcd, opj_tcd_image_t * img) {
-       int tileno, compno, resno, bandno, precno;//, cblkno;
+       int tileno, compno, resno, bandno, precno;/*, cblkno;*/
 
        fprintf(fd, "image {\n");
        fprintf(fd, "  tw=%d, th=%d x0=%d x1=%d y0=%d y1=%d\n", 
@@ -290,6 +291,7 @@ void tcd_malloc_encode(opj_tcd_t *tcd, opj_image_t * image, opj_cp_t * cp, int c
                                        for (i = 0; i < res->pw * res->ph * 3; i++) {
                                                band->precincts[i].imsbtree = NULL;
                                                band->precincts[i].incltree = NULL;
+                                               band->precincts[i].cblks.enc = NULL;
                                        }
                                        
                                        for (precno = 0; precno < res->pw * res->ph; precno++) {
@@ -332,8 +334,10 @@ void tcd_malloc_encode(opj_tcd_t *tcd, opj_image_t * image, opj_cp_t * cp, int c
                                                        cblk->y0 = int_max(cblkystart, prc->y0);
                                                        cblk->x1 = int_min(cblkxend, prc->x1);
                                                        cblk->y1 = int_min(cblkyend, prc->y1);
-                                                       cblk->data = (unsigned char*) opj_calloc(8192+2, sizeof(unsigned char));
+                                                       cblk->data = (unsigned char*) opj_calloc(9728+2, sizeof(unsigned char));
                                                        /* FIXME: mqc_init_enc and mqc_byteout underrun the buffer if we don't do this. Why? */
+                                                       cblk->data[0] = 0;
+                                                       cblk->data[1] = 0;
                                                        cblk->data += 2;
                                                        cblk->layers = (opj_tcd_layer_t*) opj_calloc(100, sizeof(opj_tcd_layer_t));
                                                        cblk->passes = (opj_tcd_pass_t*) opj_calloc(100, sizeof(opj_tcd_pass_t));
@@ -418,12 +422,19 @@ void tcd_init_encode(opj_tcd_t *tcd, opj_image_t * image, opj_cp_t * cp, int cur
                /* Modification of the RATE >> */
                for (j = 0; j < tcp->numlayers; j++) {
                        tcp->rates[j] = tcp->rates[j] ? 
-                                               ((float) (tile->numcomps 
-                                                               * (tile->x1 - tile->x0) 
-                                                               * (tile->y1 - tile->y0) 
-                                                               * image->comps[0].prec))/ 
-                                               (tcp->rates[j] * 8 * image->comps[0].dx * image->comps[0].dy) 
-                                               : 0;
+                               cp->tp_on ? 
+                                       (((float) (tile->numcomps 
+                                       * (tile->x1 - tile->x0) 
+                                       * (tile->y1 - tile->y0)
+                                       * image->comps[0].prec))
+                                       /(tcp->rates[j] * 8 * image->comps[0].dx * image->comps[0].dy)) - (((tcd->cur_totnum_tp - 1) * 14 )/ tcp->numlayers)
+                                       :
+                               ((float) (tile->numcomps 
+                                       * (tile->x1 - tile->x0) 
+                                       * (tile->y1 - tile->y0) 
+                                       * image->comps[0].prec))/ 
+                                       (tcp->rates[j] * 8 * image->comps[0].dx * image->comps[0].dy)
+                                       : 0;
 
                        if (tcp->rates[j]) {
                                if (j && tcp->rates[j] < tcp->rates[j - 1] + 10) {
@@ -584,7 +595,11 @@ void tcd_init_encode(opj_tcd_t *tcd, opj_image_t * image, opj_cp_t * cp, int cur
                                                        cblk->y0 = int_max(cblkystart, prc->y0);
                                                        cblk->x1 = int_min(cblkxend, prc->x1);
                                                        cblk->y1 = int_min(cblkyend, prc->y1);
-                                                       cblk->data = (unsigned char*) opj_calloc(8192, sizeof(unsigned char));
+                                                       cblk->data = (unsigned char*) opj_calloc(8192+2, sizeof(unsigned char));
+                                                       /* FIXME: mqc_init_enc and mqc_byteout underrun the buffer if we don't do this. Why? */
+                                                       cblk->data[0] = 0;
+                                                       cblk->data[1] = 0;
+                                                       cblk->data += 2;
                                                        cblk->layers = (opj_tcd_layer_t*) opj_calloc(100, sizeof(opj_tcd_layer_t));
                                                        cblk->passes = (opj_tcd_pass_t*) opj_calloc(100, sizeof(opj_tcd_pass_t));
                                                }
@@ -604,7 +619,7 @@ void tcd_malloc_decode(opj_tcd_t *tcd, opj_image_t * image, opj_cp_t * cp) {
        tcd->image = image;
        tcd->tcd_image->tw = cp->tw;
        tcd->tcd_image->th = cp->th;
-       tcd->tcd_image->tiles = (opj_tcd_tile_t *) opj_malloc(cp->tw * cp->th * sizeof(opj_tcd_tile_t));
+    tcd->tcd_image->tiles = (opj_tcd_tile_t *) opj_calloc(cp->tw * cp->th, sizeof(opj_tcd_tile_t));
 
        /* 
        Allocate place to store the decoded data = final image
@@ -647,7 +662,7 @@ void tcd_malloc_decode(opj_tcd_t *tcd, opj_image_t * image, opj_cp_t * cp) {
                        tilec->y1 = int_ceildiv(tile->y1, image->comps[i].dy);
 
                        x0 = j == 0 ? tilec->x0 : int_min(x0, (unsigned int) tilec->x0);
-                       y0 = j == 0 ? tilec->y0 : int_min(y0,   (unsigned int) tilec->x0);
+                       y0 = j == 0 ? tilec->y0 : int_min(y0,   (unsigned int) tilec->y0);
                        x1 = j == 0 ? tilec->x1 : int_max(x1,   (unsigned int) tilec->x1);
                        y1 = j == 0 ? tilec->y1 : int_max(y1,   (unsigned int) tilec->y1);
                }
@@ -667,6 +682,8 @@ void tcd_malloc_decode_tile(opj_tcd_t *tcd, opj_image_t * image, opj_cp_t * cp,
        opj_tcp_t *tcp;
        opj_tcd_tile_t *tile;
 
+       OPJ_ARG_NOT_USED(cstr_info);
+
        tcd->cp = cp;
        
        tcp = &(cp->tcps[cp->tileno[tileno]]);
@@ -988,7 +1005,7 @@ void tcd_makelayer(opj_tcd_t *tcd, int layno, double thresh, int final) {
        }
 }
 
-bool tcd_rateallocate(opj_tcd_t *tcd, unsigned char *dest, int len, opj_codestream_info_t *cstr_info) {
+opj_bool tcd_rateallocate(opj_tcd_t *tcd, unsigned char *dest, int len, opj_codestream_info_t *cstr_info) {
        int compno, resno, bandno, precno, cblkno, passno, layno;
        double min, max;
        double cumdisto[100];   /* fixed_quality */
@@ -1085,7 +1102,7 @@ bool tcd_rateallocate(opj_tcd_t *tcd, unsigned char *dest, int len, opj_codestre
                        opj_t2_t *t2 = t2_create(tcd->cinfo, tcd->image, cp);
                        double thresh = 0;
 
-                       for (i = 0; i < 32; i++) {
+                       for (i = 0; i < 128; i++) {
                                int l = 0;
                                double distoachieved = 0;       /* fixed_quality */
                                thresh = (lo + hi) / 2;
@@ -1094,7 +1111,7 @@ bool tcd_rateallocate(opj_tcd_t *tcd, unsigned char *dest, int len, opj_codestre
                                
                                if (cp->fixed_quality) {        /* fixed_quality */
                                        if(cp->cinema){
-                                               l = t2_encode_packets(t2,tcd->tcd_tileno, tcd_tile, layno + 1, dest, maxlen, cstr_info,tcd->cur_tp_num,tcd->tp_pos,tcd->cur_pino,THRESH_CALC);
+                                               l = t2_encode_packets(t2,tcd->tcd_tileno, tcd_tile, layno + 1, dest, maxlen, cstr_info,tcd->cur_tp_num,tcd->tp_pos,tcd->cur_pino,THRESH_CALC, tcd->cur_totnum_tp);
                                                if (l == -999) {
                                                        lo = thresh;
                                                        continue;
@@ -1120,7 +1137,7 @@ bool tcd_rateallocate(opj_tcd_t *tcd, unsigned char *dest, int len, opj_codestre
                                                lo = thresh;
                                        }
                                } else {
-                                       l = t2_encode_packets(t2, tcd->tcd_tileno, tcd_tile, layno + 1, dest, maxlen, cstr_info,tcd->cur_tp_num,tcd->tp_pos,tcd->cur_pino,THRESH_CALC);
+                                       l = t2_encode_packets(t2, tcd->tcd_tileno, tcd_tile, layno + 1, dest, maxlen, cstr_info,tcd->cur_tp_num,tcd->tp_pos,tcd->cur_pino,THRESH_CALC, tcd->cur_totnum_tp);
                                        /* TODO: what to do with l ??? seek / tell ??? */
                                        /* opj_event_msg(tcd->cinfo, EVT_INFO, "rate alloc: len=%d, max=%d\n", l, maxlen); */
                                        if (l == -999) {
@@ -1140,7 +1157,7 @@ bool tcd_rateallocate(opj_tcd_t *tcd, unsigned char *dest, int len, opj_codestre
                }
                
                if (!success) {
-                       return false;
+                       return OPJ_FALSE;
                }
                
                if(cstr_info) { /* Threshold for Marcela Index */
@@ -1152,7 +1169,7 @@ bool tcd_rateallocate(opj_tcd_t *tcd, unsigned char *dest, int len, opj_codestre
                cumdisto[layno] = (layno == 0) ? tcd_tile->distolayer[0] : (cumdisto[layno - 1] + tcd_tile->distolayer[layno]); 
        }
 
-       return true;
+       return OPJ_TRUE;
 }
 
 int tcd_encode_tile(opj_tcd_t *tcd, int tileno, unsigned char *dest, int len, opj_codestream_info_t *cstr_info) {
@@ -1284,7 +1301,7 @@ int tcd_encode_tile(opj_tcd_t *tcd, int tileno, unsigned char *dest, int len, op
        }
 
        t2 = t2_create(tcd->cinfo, image, cp);
-       l = t2_encode_packets(t2,tileno, tile, tcd_tcp->numlayers, dest, len, cstr_info,tcd->tp_num,tcd->tp_pos,tcd->cur_pino,FINAL_PASS);
+       l = t2_encode_packets(t2,tileno, tile, tcd_tcp->numlayers, dest, len, cstr_info,tcd->tp_num,tcd->tp_pos,tcd->cur_pino,FINAL_PASS,tcd->cur_totnum_tp);
        t2_destroy(t2);
        
        /*---------------CLEAN-------------------*/
@@ -1304,7 +1321,7 @@ int tcd_encode_tile(opj_tcd_t *tcd, int tileno, unsigned char *dest, int len, op
        return l;
 }
 
-bool tcd_decode_tile(opj_tcd_t *tcd, unsigned char *src, int len, int tileno, opj_codestream_info_t *cstr_info) {
+opj_bool tcd_decode_tile(opj_tcd_t *tcd, unsigned char *src, int len, int tileno, opj_codestream_info_t *cstr_info) {
        int l;
        int compno;
        int eof = 0;
@@ -1340,7 +1357,7 @@ bool tcd_decode_tile(opj_tcd_t *tcd, unsigned char *src, int len, int tileno, op
                                }
                                else {
                                        cstr_info->tile[tileno].pdx[resno] = 15;
-                                       cstr_info->tile[tileno].pdx[resno] = 15;
+                                       cstr_info->tile[tileno].pdy[resno] = 15;
                                }
                        }
                }
@@ -1364,10 +1381,23 @@ bool tcd_decode_tile(opj_tcd_t *tcd, unsigned char *src, int len, int tileno, op
        
        t1_time = opj_clock();  /* time needed to decode a tile */
        t1 = t1_create(tcd->cinfo);
+    if (t1 == NULL)
+    {
+        opj_event_msg(tcd->cinfo, EVT_ERROR, "Out of memory\n");
+        t1_destroy(t1);
+        return OPJ_FALSE;
+    }
+
        for (compno = 0; compno < tile->numcomps; ++compno) {
                opj_tcd_tilecomp_t* tilec = &tile->comps[compno];
                /* The +3 is headroom required by the vectorized DWT */
                tilec->data = (int*) opj_aligned_malloc((((tilec->x1 - tilec->x0) * (tilec->y1 - tilec->y0))+3) * sizeof(int));
+        if (tilec->data == NULL)
+        {
+            opj_event_msg(tcd->cinfo, EVT_ERROR, "Out of memory\n");
+            return OPJ_FALSE;
+        }
+
                t1_decode_cblks(t1, tilec, &tcd->tcp->tccps[compno]);
        }
        t1_destroy(t1);
@@ -1387,7 +1417,7 @@ bool tcd_decode_tile(opj_tcd_t *tcd, unsigned char *src, int len, int tileno, op
                        if (tcd->image->comps[compno].resno_decoded < 0) {                              
                                opj_event_msg(tcd->cinfo, EVT_ERROR, "Error decoding tile. The number of resolutions to remove [%d+1] is higher than the number "
                                        " of resolutions in the original codestream [%d]\nModify the cp_reduce parameter.\n", tcd->cp->reduce, tile->comps[compno].numresolutions);
-                               return false;
+                               return OPJ_FALSE;
                        }
                }
 
@@ -1407,18 +1437,23 @@ bool tcd_decode_tile(opj_tcd_t *tcd, unsigned char *src, int len, int tileno, op
 
        if (tcd->tcp->mct) {
                int n = (tile->comps[0].x1 - tile->comps[0].x0) * (tile->comps[0].y1 - tile->comps[0].y0);
-               if (tcd->tcp->tccps[0].qmfbid == 1) {
-                       mct_decode(
-                                       tile->comps[0].data,
-                                       tile->comps[1].data,
-                                       tile->comps[2].data, 
-                                       n);
-               } else {
-                       mct_decode_real(
-                                       (float*)tile->comps[0].data,
-                                       (float*)tile->comps[1].data,
-                                       (float*)tile->comps[2].data, 
-                                       n);
+
+               if (tile->numcomps >= 3 ){
+                       if (tcd->tcp->tccps[0].qmfbid == 1) {
+                               mct_decode(
+                                               tile->comps[0].data,
+                                               tile->comps[1].data,
+                                               tile->comps[2].data,
+                                               n);
+                       } else {
+                               mct_decode_real(
+                                               (float*)tile->comps[0].data,
+                                               (float*)tile->comps[1].data,
+                                               (float*)tile->comps[2].data,
+                                               n);
+                       }
+               } else{
+                       opj_event_msg(tcd->cinfo, EVT_WARNING,"Number of components (%d) is inconsistent with a MCT. Skip the MCT step.\n",tile->numcomps);
                }
        }
 
@@ -1442,6 +1477,11 @@ bool tcd_decode_tile(opj_tcd_t *tcd, unsigned char *src, int len, int tileno, op
                if(!imagec->data){
                        imagec->data = (int*) opj_malloc(imagec->w * imagec->h * sizeof(int));
                }
+        if (!imagec->data)
+        {
+            opj_event_msg(tcd->cinfo, EVT_ERROR, "Out of memory\n");
+            return OPJ_FALSE;
+        }
                if(tcd->tcp->tccps[compno].qmfbid == 1) {
                        for(j = res->y0; j < res->y1; ++j) {
                                for(i = res->x0; i < res->x1; ++i) {
@@ -1467,39 +1507,60 @@ bool tcd_decode_tile(opj_tcd_t *tcd, unsigned char *src, int len, int tileno, op
        opj_event_msg(tcd->cinfo, EVT_INFO, "- tile decoded in %f s\n", tile_time);
 
        if (eof) {
-               return false;
+               return OPJ_FALSE;
        }
        
-       return true;
+       return OPJ_TRUE;
 }
 
 void tcd_free_decode(opj_tcd_t *tcd) {
        opj_tcd_image_t *tcd_image = tcd->tcd_image;    
+    int i = 0;
+    for (i = 0; i < tcd_image->tw * tcd_image->th; i++)
+    {
+        tcd_free_decode_tile(tcd, i);
+    }
+
        opj_free(tcd_image->tiles);
 }
 
 void tcd_free_decode_tile(opj_tcd_t *tcd, int tileno) {
-       int compno,resno,bandno,precno;
+    int compno,resno,bandno,precno,cblkno;
 
        opj_tcd_image_t *tcd_image = tcd->tcd_image;
 
        opj_tcd_tile_t *tile = &tcd_image->tiles[tileno];
-       for (compno = 0; compno < tile->numcomps; compno++) {
-               opj_tcd_tilecomp_t *tilec = &tile->comps[compno];
-               for (resno = 0; resno < tilec->numresolutions; resno++) {
-                       opj_tcd_resolution_t *res = &tilec->resolutions[resno];
-                       for (bandno = 0; bandno < res->numbands; bandno++) {
-                               opj_tcd_band_t *band = &res->bands[bandno];
-                               for (precno = 0; precno < res->ph * res->pw; precno++) {
-                                       opj_tcd_precinct_t *prec = &band->precincts[precno];
-                                       if (prec->imsbtree != NULL) tgt_destroy(prec->imsbtree);
-                                       if (prec->incltree != NULL) tgt_destroy(prec->incltree);
+    if (tile->comps != NULL) {
+        for (compno = 0; compno < tile->numcomps; compno++) {
+            opj_tcd_tilecomp_t *tilec = &tile->comps[compno];
+            for (resno = 0; resno < tilec->numresolutions; resno++) {
+                opj_tcd_resolution_t *res = &tilec->resolutions[resno];
+                for (bandno = 0; bandno < res->numbands; bandno++) {
+                    opj_tcd_band_t *band = &res->bands[bandno];
+                    for (precno = 0; precno < res->ph * res->pw; precno++) {
+                        opj_tcd_precinct_t *prec = &band->precincts[precno];
+                        if (prec->cblks.dec != NULL) {
+                            for (cblkno = 0; cblkno < prec->cw * prec->ch; ++cblkno) {
+                                opj_tcd_cblk_dec_t* cblk = &prec->cblks.dec[cblkno];
+                                opj_free(cblk->data);
+                                opj_free(cblk->segs);
+                            }
+                            opj_free(prec->cblks.dec);
+                        }
+                        if (prec->imsbtree != NULL) tgt_destroy(prec->imsbtree);
+                        if (prec->incltree != NULL) tgt_destroy(prec->incltree);
+
+
+                    }
+                    opj_free(band->precincts);
                                }
-                               opj_free(band->precincts);
                        }
+            opj_free(tilec->resolutions);
                }
-               opj_free(tilec->resolutions);
+        opj_free(tile->comps);
+        tile->comps = NULL;
        }
-       opj_free(tile->comps);
 }
 
+
+