From a419afd69b0866c5ed2a4d2a301a8394d466e480 Mon Sep 17 00:00:00 2001
From: Su_Laus <sulau@freenet.de>
Date: Sat, 4 Feb 2023 23:24:21 +0100
Subject: tiffcrop correctly update buffersize after rotateImage() fix#520  --
 enlarge buffsize and check integer overflow within rotateImage().

---
 tools/tiffcrop.c | 41 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 36 insertions(+), 5 deletions(-)

diff --git a/tools/tiffcrop.c b/tools/tiffcrop.c
index 3ea42261..bbaec9e7 100644
--- a/tools/tiffcrop.c
+++ b/tools/tiffcrop.c
@@ -8524,7 +8524,8 @@ rotateImage(uint16 rotation, struct image_data *image, uint32 *img_width,
   uint32   bytes_per_pixel, bytes_per_sample;
   uint32   row, rowsize, src_offset, dst_offset;
   uint32   i, col, width, length;
-  uint32   colsize, buffsize, col_offset, pix_offset;
+  uint32   colsize, col_offset, pix_offset;
+  tmsize_t buffsize;
   unsigned char *ibuff;
   unsigned char *src;
   unsigned char *dst;
@@ -8537,12 +8538,41 @@ rotateImage(uint16 rotation, struct image_data *image, uint32 *img_width,
   spp = image->spp;
   bps = image->bps;
 
+  if ((spp != 0 && bps != 0 &&
+       width > (uint32_t)((UINT32_MAX - 7) / spp / bps)) ||
+      (spp != 0 && bps != 0 &&
+       length > (uint32_t)((UINT32_MAX - 7) / spp / bps)))
+  {
+      TIFFError("rotateImage", "Integer overflow detected.");
+      return (-1);
+  }
+
   rowsize = ((bps * spp * width) + 7) / 8;
   colsize = ((bps * spp * length) + 7) / 8;
   if ((colsize * width) > (rowsize * length))
-    buffsize = (colsize + 1) * width;
-  else
-    buffsize = (rowsize + 1) * length;
+    {
+        if (((tmsize_t)colsize + 1) != 0 &&
+            (tmsize_t)width > ((TIFF_TMSIZE_T_MAX - NUM_BUFF_OVERSIZE_BYTES) /
+                               ((tmsize_t)colsize + 1)))
+        {
+            TIFFError("rotateImage",
+                      "Integer overflow when calculating buffer size.");
+            return (-1);
+        }
+        buffsize = ((tmsize_t)colsize + 1) * width;
+    }
+    else
+    {
+        if (((tmsize_t)rowsize + 1) != 0 &&
+            (tmsize_t)length > ((TIFF_TMSIZE_T_MAX - NUM_BUFF_OVERSIZE_BYTES) /
+                                ((tmsize_t)rowsize + 1)))
+        {
+            TIFFError("rotateImage",
+                      "Integer overflow when calculating buffer size.");
+            return (-1);
+        }
+        buffsize = (rowsize + 1) * length;
+    }
 
   bytes_per_sample = (bps + 7) / 8;
   bytes_per_pixel  = ((bps * spp) + 7) / 8;
@@ -8565,7 +8595,8 @@ rotateImage(uint16 rotation, struct image_data *image, uint32 *img_width,
   /* Add 3 padding bytes for extractContigSamplesShifted32bits */
   if (!(rbuff = (unsigned char *)limitMalloc(buffsize + NUM_BUFF_OVERSIZE_BYTES)))
     {
-    TIFFError("rotateImage", "Unable to allocate rotation buffer of %1u bytes", buffsize + NUM_BUFF_OVERSIZE_BYTES);
+    TIFFError("rotateImage", "Unable to allocate rotation buffer of " TIFF_SSIZE_FORMAT
+                  " bytes ", buffsize + NUM_BUFF_OVERSIZE_BYTES);
     return (-1);
     }
   _TIFFmemset(rbuff, '\0', buffsize + NUM_BUFF_OVERSIZE_BYTES);
-- 
2.30.2