GraphicsMagick: ReadSUNImage(): Enlarge RLE output buffer in ord...

changeset fe4699f9f06c in /hg/GraphicsMagick
details: http://hg.GraphicsMagick.org/hg/GraphicsMagick?cmd=changeset;node=fe4699f9f06c
summary: ReadSUNImage(): Enlarge RLE output buffer in order to avoid buffer overflow.

diffstat:

 ChangeLog                        |   11 +
 coders/sun.c                     |  773 ++++++++++++++++++++------------------
 coders/xpm.c                     |    4 +-
 magick/utility.c                 |    4 +
 tests/input_truecolor_1x266.miff |    5 +
 tests/rwfile.tap                 |    2 +-
 www/Changelog.html               |   12 +
 7 files changed, 435 insertions(+), 376 deletions(-)

diffs (truncated from 923 to 500 lines):

diff -r 758a0568bfc3 -r fe4699f9f06c ChangeLog
--- a/ChangeLog	Sun Jan 01 09:36:12 2023 -0600
+++ b/ChangeLog	Sun Jan 01 15:57:07 2023 -0600
@@ -1,3 +1,14 @@
+2023-01-01  Bob Friesenhahn  <[email protected]>
+
+	* magick/utility.c (GetMagickGeometry): Assure that width and
+	height are not scaled down to zero since it is an invalid value.
+
+	* coders/sun.c (ReadSUNImage): Enlarge RLE output buffer in order
+	to avoid buffer overflow.  Addresses oss-fuzz 54716
+	"graphicsmagick:coder_RAS_fuzzer: Heap-buffer-overflow in
+	ReadSUNImage", which is due to a new problem added since the
+	1.3.39 release.
+
 2023-01-01 Fojtik Jaroslav  <[email protected]>
 
 	* jp2/* Update lib jasper to 2.0.0.
diff -r 758a0568bfc3 -r fe4699f9f06c coders/sun.c
--- a/coders/sun.c	Sun Jan 01 09:36:12 2023 -0600
+++ b/coders/sun.c	Sun Jan 01 15:57:07 2023 -0600
@@ -303,6 +303,7 @@
     bytes_per_image,
     bytes_per_line,
     count,
+    pad,
     sun_data_length;
 
   SUNInfo
@@ -339,422 +340,448 @@
   (void) memset(&sun_info,0,sizeof(sun_info));
   sun_info.magic=ReadBlobMSBLong(image);
   do
-  {
-    /*
-      Verify SUN identifier.
-    */
-    if (sun_info.magic != 0x59a66a95)
-      ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
-    sun_info.width=ReadBlobMSBLong(image);
-    sun_info.height=ReadBlobMSBLong(image);
-    sun_info.depth=ReadBlobMSBLong(image);
-    sun_info.length=ReadBlobMSBLong(image);
-    sun_info.type=ReadBlobMSBLong(image);
-    sun_info.maptype=ReadBlobMSBLong(image);
-    sun_info.maplength=ReadBlobMSBLong(image);
-    if (logging)
-      LogSUNInfo(&sun_info);
-    if (EOFBlob(image))
-      ThrowReaderException(CorruptImageError,UnexpectedEndOfFile,image);
-    /*
-      Verify that width, height, depth, and length are not zero
-    */
-    if ((sun_info.width == 0) || (sun_info.height == 0) ||
-        (sun_info.depth == 0) || (sun_info.length == 0))
-      ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+    {
+      /*
+        Verify SUN identifier.
+      */
+      if (sun_info.magic != 0x59a66a95)
+        ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+      sun_info.width=ReadBlobMSBLong(image);
+      sun_info.height=ReadBlobMSBLong(image);
+      sun_info.depth=ReadBlobMSBLong(image);
+      sun_info.length=ReadBlobMSBLong(image);
+      sun_info.type=ReadBlobMSBLong(image);
+      sun_info.maptype=ReadBlobMSBLong(image);
+      sun_info.maplength=ReadBlobMSBLong(image);
+      if (logging)
+        LogSUNInfo(&sun_info);
+      if (EOFBlob(image))
+        ThrowReaderException(CorruptImageError,UnexpectedEndOfFile,image);
+      /*
+        Verify that width, height, depth, and length are not zero
+      */
+      if ((sun_info.width == 0) || (sun_info.height == 0) ||
+          (sun_info.depth == 0) || (sun_info.length == 0))
+        ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+
+      /*
+        Verify that header values are in positive numeric range of a
+        32-bit 'int' even though we store them in an unsigned value.
+      */
+      if ((sun_info.magic | sun_info.width | sun_info.height | sun_info.depth |
+           sun_info.type | sun_info.maptype | sun_info.maplength) & (1U << 31))
+        ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+      /*
+        Verify that we support the image sub-type
+      */
+      if ((sun_info.type != RT_STANDARD) &&
+          (sun_info.type != RT_ENCODED) &&
+          (sun_info.type != RT_FORMAT_RGB))
+        ThrowReaderException(CoderError,DataEncodingSchemeIsNotSupported,image);
+      /*
+        Verify that we support the colormap type
+      */
+      if ((sun_info.maptype != RMT_NONE) &&
+          (sun_info.maptype != RMT_EQUAL_RGB))
+        ThrowReaderException(CoderError,ColormapTypeNotSupported,image);
+      /*
+        Insist that map length is zero if there is no colormap.
+      */
+      if ((sun_info.maptype == RMT_NONE) && (sun_info.maplength != 0))
+        ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+      /*
+        Insist on a supported depth
+      */
+      if ((sun_info.depth != 1) &&
+          (sun_info.depth != 8) &&
+          (sun_info.depth != 24) &&
+          (sun_info.depth != 32))
+        ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+
+      image->columns=sun_info.width;
+      image->rows=sun_info.height;
+      if (((unsigned long) ((long) image->columns) != image->columns) ||
+          ((unsigned long) ((long) image->rows) != image->rows))
+        ThrowReaderException(CoderError,ImageColumnOrRowSizeIsNotSupported,image);
+      if (CheckImagePixelLimits(image, exception) != MagickPass)
+        ThrowReaderException(ResourceLimitError,ImagePixelLimitExceeded,image);
+      image->depth=sun_info.depth <= 8 ? 8 : QuantumDepth;
+      if (sun_info.depth < 24)
+        {
+          image->colors=sun_info.maplength;
+          if (sun_info.maptype == RMT_NONE)
+            image->colors=1U << sun_info.depth;
+          if (sun_info.maptype == RMT_EQUAL_RGB)
+            image->colors=sun_info.maplength/3U;
+        }
 
-    /*
-      Verify that header values are in positive numeric range of a
-      32-bit 'int' even though we store them in an unsigned value.
-    */
-    if ((sun_info.magic | sun_info.width | sun_info.height | sun_info.depth |
-         sun_info.type | sun_info.maptype | sun_info.maplength) & (1U << 31))
-      ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
-    /*
-      Verify that we support the image sub-type
-    */
-    if ((sun_info.type != RT_STANDARD) &&
-        (sun_info.type != RT_ENCODED) &&
-        (sun_info.type != RT_FORMAT_RGB))
-      ThrowReaderException(CoderError,DataEncodingSchemeIsNotSupported,image);
-    /*
-      Verify that we support the colormap type
-    */
-    if ((sun_info.maptype != RMT_NONE) &&
-        (sun_info.maptype != RMT_EQUAL_RGB))
-      ThrowReaderException(CoderError,ColormapTypeNotSupported,image);
-    /*
-      Insist that map length is zero if there is no colormap.
-    */
-    if ((sun_info.maptype == RMT_NONE) && (sun_info.maplength != 0))
-      ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
-    /*
-      Insist on a supported depth
-    */
-    if ((sun_info.depth != 1) &&
-        (sun_info.depth != 8) &&
-        (sun_info.depth != 24) &&
-        (sun_info.depth != 32))
-      ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+      switch (sun_info.maptype)
+        {
+        case RMT_NONE:
+          {
+            if (sun_info.depth < 24)
+              {
+                /*
+                  Create linear color ramp.
+                */
+                if (!AllocateImageColormap(image,image->colors))
+                  ThrowReaderException(ResourceLimitError,MemoryAllocationFailed,
+                                       image);
+                if (logging)
+                  (void) LogMagickEvent(CoderEvent,GetMagickModule(),
+                                        "Allocated colormap with %u colors", image->colors);
+              }
+            break;
+          }
+        case RMT_EQUAL_RGB:
+          {
+            unsigned char
+              *sun_colormap;
 
-    image->columns=sun_info.width;
-    image->rows=sun_info.height;
-    if (((unsigned long) ((long) image->columns) != image->columns) ||
-        ((unsigned long) ((long) image->rows) != image->rows))
-      ThrowReaderException(CoderError,ImageColumnOrRowSizeIsNotSupported,image);
-    if (CheckImagePixelLimits(image, exception) != MagickPass)
-        ThrowReaderException(ResourceLimitError,ImagePixelLimitExceeded,image);
-    image->depth=sun_info.depth <= 8 ? 8 : QuantumDepth;
-    if (sun_info.depth < 24)
-      {
-        image->colors=sun_info.maplength;
-        if (sun_info.maptype == RMT_NONE)
-          image->colors=1U << sun_info.depth;
-        if (sun_info.maptype == RMT_EQUAL_RGB)
-          image->colors=sun_info.maplength/3U;
-      }
+            /*
+              Read SUN raster colormap.
+            */
+            if (!AllocateImageColormap(image,image->colors))
+              ThrowReaderException(ResourceLimitError,MemoryAllocationFailed,
+                                   image);
+            sun_colormap=MagickAllocateResourceLimitedMemory(unsigned char *,image->colors);
+            if (sun_colormap == (unsigned char *) NULL)
+              ThrowReaderException(ResourceLimitError,MemoryAllocationFailed,
+                                   image);
+            do
+              {
+                if (ReadBlob(image,image->colors,(char *) sun_colormap) !=
+                    image->colors)
+                  {
+                    status = MagickFail;
+                    break;
+                  }
+                for (i=0; i < (long) image->colors; i++)
+                  image->colormap[i].red=ScaleCharToQuantum(sun_colormap[i]);
+                if (ReadBlob(image,image->colors,(char *) sun_colormap) !=
+                    image->colors)
+                  {
+                    status = MagickFail;
+                    break;
+                  }
+                for (i=0; i < (long) image->colors; i++)
+                  image->colormap[i].green=ScaleCharToQuantum(sun_colormap[i]);
+                if (ReadBlob(image,image->colors,(char *) sun_colormap) !=
+                    image->colors)
+                  {
+                    status = MagickFail;
+                    break;
+                  }
+                for (i=0; i < (long) image->colors; i++)
+                  image->colormap[i].blue=ScaleCharToQuantum(sun_colormap[i]);
+                break;
+              } while (1);
+            MagickFreeResourceLimitedMemory(sun_colormap);
+            if (MagickFail == status)
+              ThrowReaderException(CorruptImageError,UnexpectedEndOfFile,image);
+            if (logging)
+              (void) LogMagickEvent(CoderEvent,GetMagickModule(),
+                                    "Read colormap with %u colors", image->colors);
+            break;
+          }
+        case RMT_RAW:
+          {
+            unsigned char
+              *sun_colormap;
 
-    switch (sun_info.maptype)
-    {
-      case RMT_NONE:
-      {
-        if (sun_info.depth < 24)
-          {
             /*
-              Create linear color ramp.
+              Read SUN raster colormap.
             */
             if (!AllocateImageColormap(image,image->colors))
               ThrowReaderException(ResourceLimitError,MemoryAllocationFailed,
-                image);
+                                   image);
+            sun_colormap=MagickAllocateResourceLimitedMemory(unsigned char *,sun_info.maplength);
+            if (sun_colormap == (unsigned char *) NULL)
+              ThrowReaderException(ResourceLimitError,MemoryAllocationFailed,
+                                   image);
+            if (ReadBlob(image,sun_info.maplength,(char *) sun_colormap) !=
+                sun_info.maplength)
+              status = MagickFail;
+            MagickFreeResourceLimitedMemory(sun_colormap);
+            if (MagickFail == status)
+              ThrowReaderException(CorruptImageError,UnexpectedEndOfFile,image);
+            if (logging)
+              (void) LogMagickEvent(CoderEvent,GetMagickModule(),
+                                    "Read colormap with %u colors (length %u)",
+                                    image->colors, sun_info.maplength);
+            break;
           }
-        break;
-      }
-      case RMT_EQUAL_RGB:
-      {
-        unsigned char
-          *sun_colormap;
+        default:
+          ThrowReaderException(CoderError,ColormapTypeNotSupported,image)
+            }
+      image->matte=(sun_info.depth == 32);
+      image->columns=sun_info.width;
+      image->rows=sun_info.height;
+      image->depth=8;
+      if (sun_info.depth < 8)
+        image->depth=sun_info.depth;
 
-        /*
-          Read SUN raster colormap.
-        */
-        if (!AllocateImageColormap(image,image->colors))
-          ThrowReaderException(ResourceLimitError,MemoryAllocationFailed,
-            image);
-        sun_colormap=MagickAllocateResourceLimitedMemory(unsigned char *,image->colors);
-        if (sun_colormap == (unsigned char *) NULL)
-          ThrowReaderException(ResourceLimitError,MemoryAllocationFailed,
-                               image);
-        do
-          {
-            if (ReadBlob(image,image->colors,(char *) sun_colormap) !=
-                image->colors)
-              {
-                status = MagickFail;
-                break;
-              }
-            for (i=0; i < (long) image->colors; i++)
-              image->colormap[i].red=ScaleCharToQuantum(sun_colormap[i]);
-            if (ReadBlob(image,image->colors,(char *) sun_colormap) !=
-                image->colors)
-              {
-                status = MagickFail;
-                break;
-              }
-            for (i=0; i < (long) image->colors; i++)
-              image->colormap[i].green=ScaleCharToQuantum(sun_colormap[i]);
-            if (ReadBlob(image,image->colors,(char *) sun_colormap) !=
-                image->colors)
-              {
-                status = MagickFail;
-                break;
-              }
-            for (i=0; i < (long) image->colors; i++)
-              image->colormap[i].blue=ScaleCharToQuantum(sun_colormap[i]);
-            break;
-          } while (1);
-        MagickFreeResourceLimitedMemory(sun_colormap);
-        if (MagickFail == status)
-          ThrowReaderException(CorruptImageError,UnexpectedEndOfFile,image);
-        break;
-      }
-      case RMT_RAW:
-      {
-        unsigned char
-          *sun_colormap;
+      if (image_info->ping)
+        {
+          CloseBlob(image);
+          return(image);
+        }
+
+      /*
+        Compute bytes per line and bytes per image for an unencoded
+        image.
 
-        /*
-          Read SUN raster colormap.
-        */
-        if (!AllocateImageColormap(image,image->colors))
-          ThrowReaderException(ResourceLimitError,MemoryAllocationFailed,
-                               image);
-        sun_colormap=MagickAllocateResourceLimitedMemory(unsigned char *,sun_info.maplength);
-        if (sun_colormap == (unsigned char *) NULL)
-          ThrowReaderException(ResourceLimitError,MemoryAllocationFailed,
-            image);
-        if (ReadBlob(image,sun_info.maplength,(char *) sun_colormap) !=
-            sun_info.maplength)
-          status = MagickFail;
-        MagickFreeResourceLimitedMemory(sun_colormap);
-        if (MagickFail == status)
-          ThrowReaderException(CorruptImageError,UnexpectedEndOfFile,image);
-        break;
-      }
-      default:
-        ThrowReaderException(CoderError,ColormapTypeNotSupported,image)
-    }
-    image->matte=(sun_info.depth == 32);
-    image->columns=sun_info.width;
-    image->rows=sun_info.height;
-    image->depth=8;
-    if (sun_info.depth < 8)
-      image->depth=sun_info.depth;
+        "The width of a scan line is always 16-bits, padded when necessary."
+      */
+      bytes_per_line=MagickArraySize(sun_info.width,sun_info.depth);
+      if (bytes_per_line == 0)
+        ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+
+      pad = sun_info.depth == 1 ? 15U : 7U;  /* Pad */
+      bytes_per_line += pad;
 
-    if (image_info->ping)
-      {
-        CloseBlob(image);
-        return(image);
-      }
+      if (bytes_per_line != ((size_t) sun_info.width*sun_info.depth+pad))
+        ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
 
-    /*
-      Compute bytes per line and bytes per image for an unencoded
-      image.
+      bytes_per_line /= 8U;
+      if (bytes_per_line == 0)
+        ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+
+      bytes_per_image=MagickArraySize(sun_info.height,bytes_per_line);
+      if (bytes_per_image == 0)
+        ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
 
-      "The width of a scan line is always 16-bits, padded when necessary."
-    */
-    bytes_per_line=MagickArraySize(sun_info.width,sun_info.depth);
-    if (bytes_per_line == 0)
-      ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
-
-    bytes_per_line += sun_info.depth == 1 ? 15U : 7U; /* Pad */
-    bytes_per_line /= 8U;
-    if (bytes_per_line == 0)
-      ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
-
-    bytes_per_image=MagickArraySize(sun_info.height,bytes_per_line);
-    if (bytes_per_image == 0)
-      ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
-
-    if (sun_info.type == RT_ENCODED)
-      sun_data_length=(size_t) sun_info.length;
-    else
-      sun_data_length=bytes_per_image;
+      if (sun_info.type == RT_ENCODED)
+        sun_data_length=(size_t) sun_info.length;
+      else
+        sun_data_length=bytes_per_image;
 
-    /*
-      Verify that data length claimed by header is supported by file size
-    */
-    if (sun_info.type == RT_ENCODED)
-      {
-        if (sun_data_length < bytes_per_image/255U)
-          ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
-      }
-    if (BlobIsSeekable(image))
-      {
-        const magick_off_t file_size = GetBlobSize(image);
-        const magick_off_t current_offset = TellBlob(image);
-        if ((file_size > 0) &&
-            (current_offset > 0) &&
-            (file_size >= current_offset))
+      /*
+        Verify that data length claimed by header is supported by file size
+      */
+      if (sun_info.type == RT_ENCODED)
+        {
+          if (sun_data_length < bytes_per_image/255U)
+            ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+        }
+      if (BlobIsSeekable(image))
         {
-          const magick_off_t remaining = file_size-current_offset;
+          const magick_off_t file_size = GetBlobSize(image);
+          const magick_off_t current_offset = TellBlob(image);
+          if ((file_size > 0) &&
+              (current_offset > 0) &&
+              (file_size >= current_offset))
+            {
+              const magick_off_t remaining = file_size-current_offset;
 
-          if ((remaining == 0) || (remaining < (magick_off_t) sun_info.length))
-            {
-              ThrowReaderException(CorruptImageError,UnexpectedEndOfFile,image);
+              if ((remaining == 0) || (remaining < (magick_off_t) sun_info.length))
+                {
+                  ThrowReaderException(CorruptImageError,UnexpectedEndOfFile,image);
+                }
             }
         }
-      }
 
-    /*
-      Read raster data into allocated buffer.
-    */
-    sun_data=MagickAllocateResourceLimitedMemory(unsigned char *,sun_info.length);
-    if (sun_data == (unsigned char *) NULL)
-      ThrowReaderException(ResourceLimitError,MemoryAllocationFailed,image);
-    if ((count=ReadBlob(image,sun_info.length,(char *) sun_data))
-        != sun_info.length)
-      {
-        MagickFreeResourceLimitedMemory(sun_data);
-        ThrowReaderException(CorruptImageError,UnableToReadImageData,image);
-      }
-    sun_pixels=sun_data;
-    if (sun_info.type == RT_ENCODED)
-      {
-        /*
-          Read run-length encoded raster pixels (padded to 16-bit boundary).
-        */
-        sun_pixels=MagickAllocateResourceLimitedMemory(unsigned char *,bytes_per_image);