openjp3d: Int overflow fixed (#1159)

When compressing a lot of slices (starting from 44 FullHD slices with 3 8bit components in our experiments) the rate values are high enough to cause an int overflow that leads to negative lengths and wrong results. The cast happens too late.

diff --git a/src/lib/openjp3d/tcd.c b/src/lib/openjp3d/tcd.c
index 013a9f1ac59d29ce1848ea3b1f30518e0b67b6a3..714a0565d5dc13840fe94632a03fa73d6df388c9 100644 (file)
--- a/src/lib/openjp3d/tcd.c
+++ b/src/lib/openjp3d/tcd.c
@@ -226,10 +226,10 @@ void tcd_malloc_encode(opj_tcd_t *tcd, opj_volume_t * volume, opj_cp_t * cp,
         if (tcp->rates[j] <= 1) {
             tcp->rates[j] = 0;
         } else {
-            float num = (float)(tile->numcomps * (tile->x1 - tile->x0) *
-                                (tile->y1 - tile->y0) * (tile->z1 - tile->z0) * volume->comps[0].prec);
-            float den = (float)(8 * volume->comps[0].dx * volume->comps[0].dy *
-                                volume->comps[0].dz);
+            float num = (float)tile->numcomps * (tile->x1 - tile->x0) *
+                        (tile->y1 - tile->y0) * (tile->z1 - tile->z0) * volume->comps[0].prec;
+            float den = 8.0f * volume->comps[0].dx * volume->comps[0].dy *
+                        volume->comps[0].dz;
             den = tcp->rates[j] * den;
             tcp->rates[j] = (num + den - 1) / den;
         }
@@ -537,10 +537,10 @@ void tcd_init_encode(opj_tcd_t *tcd, opj_volume_t * volume, opj_cp_t * cp,
         if (tcp->rates[j] <= 1) {
             tcp->rates[j] = 0;
         } else {
-            float num = (float)(tile->numcomps * (tile->x1 - tile->x0) *
-                                (tile->y1 - tile->y0) * (tile->z1 - tile->z0) * volume->comps[0].prec);
-            float den = (float)(8 * volume->comps[0].dx * volume->comps[0].dy *
-                                volume->comps[0].dz);
+            float num = (float)tile->numcomps * (tile->x1 - tile->x0) *
+                        (tile->y1 - tile->y0) * (tile->z1 - tile->z0) * volume->comps[0].prec;
+            float den = 8.0f * volume->comps[0].dx * volume->comps[0].dy *
+                        volume->comps[0].dz;
             den = tcp->rates[j] * den;
             tcp->rates[j] = (num + den - 1) / den;
         }