CVE-2016-10057: Improper Restriction of Operations within the Bounds of a Memory Buffer (CWE-119)
value: Buffer overflow in the WriteGROUP4Image function in coders/tiff.c in ImageMagick before 6.9.5-8 allows remote attackers to cause a denial of service (application crash) or have other unspecified impact via a crafted file.
CWE description: The product performs operations on a memory buffer, but it reads from or writes to a memory location outside the buffer's intended boundary. This may result in read or write operations on unexpected memory locations that could be linked to other variables, data structures, or internal program data.
CVSS v3 base score: 7.8
Published: 2017-03-23T17:59Z
Repository: https://github.com/ImageMagick/ImageMagick
Language: C
Files changed: ChangeLog, coders/map.c, coders/pdb.c, coders/sixel.c, coders/tiff.c
Commit date: 2016-08-23 17:42:10 -0400
Commit message: Prevent buffer overflow in SIXEL, PDB, MAP, and CALS coders (bug report from Donghai Zhu)

Diff: diff --git a/ChangeLog b/ChangeLog
index ff89b2021c..12c8c95f0f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,10 @@
 2016-08-15  7.0.2-10 Cristy  <quetzlzacatenango@image...>
   * Prevent buffer overflow in BMP & SGI coders (bug report from
     pwchen&rayzhong of tencent).
+  * Prevent buffer overflow in SIXEL, PDB, MAP, and CALS coders (bug report
+    from Donghai Zhu).
 
 2016-08-14  7.0.2-9 Cristy  <quetzlzacatenango@image...>
   * Release ImageMagick version 7.0.2-9, GIT revision 18707:2c02f09:20160814.
 
 2016-08-14  7.0.2-9 Cristy  <quetzlzacatenango@image...>
diff --git a/coders/map.c b/coders/map.c
index 5b6e5749f2..bd7f03501c 100644
--- a/coders/map.c
+++ b/coders/map.c
@@ -394,26 +394,27 @@ static MagickBooleanType WriteMAPImage(const ImageInfo *image_info,Image *image,
     ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
   /*
     Write colormap to file.
   */
   q=colormap;
-  if (image->depth <= 8)
+  q=colormap;
+  if (image->colors <= 256)
     for (i=0; i < (ssize_t) image->colors; i++)
     {
-      *q++=(unsigned char) image->colormap[i].red;
-      *q++=(unsigned char) image->colormap[i].green;
-      *q++=(unsigned char) image->colormap[i].blue;
+      *q++=(unsigned char) ScaleQuantumToChar(image->colormap[i].red);
+      *q++=(unsigned char) ScaleQuantumToChar(image->colormap[i].green);
+      *q++=(unsigned char) ScaleQuantumToChar(image->colormap[i].blue);
     }
   else
     for (i=0; i < (ssize_t) image->colors; i++)
     {
-      *q++=(unsigned char) ((size_t) image->colormap[i].red >> 8);
-      *q++=(unsigned char) image->colormap[i].red;
-      *q++=(unsigned char) ((size_t) image->colormap[i].green >> 8);
-      *q++=(unsigned char) image->colormap[i].green;
-      *q++=(unsigned char) ((size_t) image->colormap[i].blue >> 8);
-      *q++=(unsigned char) image->colormap[i].blue;
+      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].red) >> 8);
+      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].red) & 0xff);
+      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].green) >> 8);
+      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].green) & 0xff);;
+      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].blue) >> 8);
+      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].blue) & 0xff);
     }
   (void) WriteBlob(image,packet_size*image->colors,colormap);
   colormap=(unsigned char *) RelinquishMagickMemory(colormap);
   /*
     Write image pixels to file.
diff --git a/coders/pdb.c b/coders/pdb.c
index ed9e3d93b9..849c1dfcdd 100644
--- a/coders/pdb.c
+++ b/coders/pdb.c
@@ -824,11 +824,11 @@ static MagickBooleanType WritePDBImage(const ImageInfo *image_info,Image *image,
   if (runlength == (unsigned char *) NULL)
     ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
   buffer=(unsigned char *) AcquireQuantumMemory(512,sizeof(*buffer));
   if (buffer == (unsigned char *) NULL)
     ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
-  packet_size=(size_t) (image->depth > 8 ? 2: 1);
+  packet_size=(size_t) (image->depth > 8 ? 2 : 1);
   scanline=(unsigned char *) AcquireQuantumMemory(image->columns,packet_size*
     sizeof(*scanline));
   if (scanline == (unsigned char *) NULL)
     ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
   if (IssRGBCompatibleColorspace(image->colorspace) == MagickFalse)
@@ -837,10 +837,11 @@ static MagickBooleanType WritePDBImage(const ImageInfo *image_info,Image *image,
     Convert to GRAY raster scanline.
   */
   quantum_info=AcquireQuantumInfo(image_info,image);
   if (quantum_info == (QuantumInfo *) NULL)
     ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
+  status=SetQuantumDepth(image,quantum_info,image->depth > 8 ? 16 : 8);
   bits=8/(int) bits_per_pixel-1;  /* start at most significant bits */
   literal=0;
   repeat=0;
   q=runlength;
   buffer[0]=0x00;
diff --git a/coders/sixel.c b/coders/sixel.c
index b9901ba730..b5f2f7bbd7 100644
--- a/coders/sixel.c
+++ b/coders/sixel.c
@@ -255,11 +255,11 @@ MagickBooleanType sixel_decode(unsigned char              /* in */  *p,
     color_index = 0;
     background_color_index = 0;
 
     imsx = 2048;
     imsy = 2048;
-    imbuf = (unsigned char *) AcquireQuantumMemory(imsx * imsy,1);
+    imbuf = (unsigned char *) AcquireQuantumMemory(imsx , imsy);
 
     if (imbuf == NULL) {
         return(MagickFalse);
     }
 
@@ -282,11 +282,11 @@ MagickBooleanType sixel_decode(unsigned char              /* in */  *p,
 
     for (; n < SIXEL_PALETTE_MAX; n++) {
         sixel_palet[n] = SIXEL_RGB(255, 255, 255);
     }
 
-    (void) ResetMagickMemory(imbuf, background_color_index, imsx * imsy);
+    (void) ResetMagickMemory(imbuf, background_color_index, (size_t) imsx * imsy);
 
     while (*p != '\0') {
         if ((p[0] == '\033' && p[1] == 'P') || *p == 0x90) {
             if (*p == '\033') {
                 p++;
@@ -356,16 +356,16 @@ MagickBooleanType sixel_decode(unsigned char              /* in */  *p,
             if (attributed_pad <= 0) attributed_pad = 1;
 
             if (imsx < attributed_ph || imsy < attributed_pv) {
                 dmsx = imsx > attributed_ph ? imsx : attributed_ph;
                 dmsy = imsy > attributed_pv ? imsy : attributed_pv;
-                dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx * dmsy,1);
+                dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx , dmsy);
                 if (dmbuf == (unsigned char *) NULL) {
                     imbuf = (unsigned char *) RelinquishMagickMemory(imbuf);
                     return (MagickFalse);
                 }
-                (void) ResetMagickMemory(dmbuf, background_color_index, dmsx * dmsy);
+                (void) ResetMagickMemory(dmbuf, background_color_index, (size_t) dmsx * dmsy);
                 for (y = 0; y < imsy; ++y) {
                     (void) CopyMagickMemory(dmbuf + dmsx * y, imbuf + imsx * y, imsx);
                 }
                 imbuf = (unsigned char *) RelinquishMagickMemory(imbuf);
                 imsx = dmsx;
@@ -430,16 +430,16 @@ MagickBooleanType sixel_decode(unsigned char              /* in */  *p,
                     ny *= 2;
                 }
 
                 dmsx = nx;
                 dmsy = ny;
-                dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx * dmsy,1);
+                dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx , dmsy);
                 if (dmbuf == (unsigned char *) NULL) {
                     imbuf = (unsigned char *) RelinquishMagickMemory(imbuf);
                     return (MagickFalse);
                 }
-                (void) ResetMagickMemory(dmbuf, background_color_index, dmsx * dmsy);
+                (void) ResetMagickMemory(dmbuf, background_color_index, (size_t) dmsx * dmsy);
                 for (y = 0; y < imsy; ++y) {
                     (void) CopyMagickMemory(dmbuf + dmsx * y, imbuf + imsx * y, imsx);
                 }
                 imbuf = (unsigned char *) RelinquishMagickMemory(imbuf);
                 imsx = dmsx;
@@ -480,11 +480,11 @@ MagickBooleanType sixel_decode(unsigned char              /* in */  *p,
                                     break;
                                 }
                                 c <<= 1;
                             }
                             for (y = posision_y + i; y < posision_y + i + n; ++y) {
-                                (void) ResetMagickMemory(imbuf + imsx * y + posision_x, color_index, repeat_count);
+                                (void) ResetMagickMemory(imbuf + (size_t) imsx * y + posision_x, color_index, repeat_count);
                             }
                             if (max_x < (posision_x + repeat_count - 1)) {
                                 max_x = posision_x + repeat_count - 1;
                             }
                             if (max_y < (posision_y + i + n - 1)) {
@@ -513,11 +513,11 @@ MagickBooleanType sixel_decode(unsigned char              /* in */  *p,
     }
 
     if (imsx > max_x || imsy > max_y) {
         dmsx = max_x;
         dmsy = max_y;
-        if ((dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx * dmsy,1)) == NULL) {
+        if ((dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx , dmsy)) == NULL) {
             imbuf = (unsigned char *) RelinquishMagickMemory(imbuf);
             return (MagickFalse);
         }
         for (y = 0; y < dmsy; ++y) {
             (void) CopyMagickMemory(dmbuf + dmsx * y, imbuf + imsx * y, dmsx);
diff --git a/coders/tiff.c b/coders/tiff.c
index fd171099d3..32ed204270 100644
--- a/coders/tiff.c
+++ b/coders/tiff.c
@@ -2491,12 +2491,12 @@ static MagickBooleanType WriteGROUP4Image(const ImageInfo *image_info,
   (void) FormatLocaleString(huffman_image->filename,MagickPathExtent,"tiff:%s",
     filename);
   (void) SetImageType(huffman_image,BilevelType,exception);
   write_info=CloneImageInfo((ImageInfo *) NULL);
   SetImageInfoFile(write_info,file);
-  (void) SetImageType(image,BilevelType,exception);
   (void) SetImageDepth(image,1,exception);
+  (void) SetImageType(image,BilevelType,exception);
   write_info->compression=Group4Compression;
   write_info->type=BilevelType;
   (void) SetImageOption(write_info,"quantum:polarity","min-is-white");
   status=WriteTIFFImage(write_info,huffman_image,exception);
   (void) fflush(file);

Vulnerable code:   if (image->depth <= 8)
      *q++=(unsigned char) image->colormap[i].red;
      *q++=(unsigned char) image->colormap[i].green;
      *q++=(unsigned char) image->colormap[i].blue;
      *q++=(unsigned char) ((size_t) image->colormap[i].red >> 8);
      *q++=(unsigned char) image->colormap[i].red;
      *q++=(unsigned char) ((size_t) image->colormap[i].green >> 8);
      *q++=(unsigned char) image->colormap[i].green;
      *q++=(unsigned char) ((size_t) image->colormap[i].blue >> 8);
      *q++=(unsigned char) image->colormap[i].blue;
  packet_size=(size_t) (image->depth > 8 ? 2: 1);
    imbuf = (unsigned char *) AcquireQuantumMemory(imsx * imsy,1);
    (void) ResetMagickMemory(imbuf, background_color_index, imsx * imsy);
                dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx * dmsy,1);
                (void) ResetMagickMemory(dmbuf, background_color_index, dmsx * dmsy);
                dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx * dmsy,1);
                (void) ResetMagickMemory(dmbuf, background_color_index, dmsx * dmsy);
                                (void) ResetMagickMemory(imbuf + imsx * y + posision_x, color_index, repeat_count);
        if ((dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx * dmsy,1)) == NULL) {
  (void) SetImageType(image,BilevelType,exception);
Fixed code:   * Prevent buffer overflow in SIXEL, PDB, MAP, and CALS coders (bug report
    from Donghai Zhu).
  q=colormap;
  if (image->colors <= 256)
      *q++=(unsigned char) ScaleQuantumToChar(image->colormap[i].red);
      *q++=(unsigned char) ScaleQuantumToChar(image->colormap[i].green);
      *q++=(unsigned char) ScaleQuantumToChar(image->colormap[i].blue);
      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].red) >> 8);
      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].red) & 0xff);
      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].green) >> 8);
      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].green) & 0xff);;
      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].blue) >> 8);
      *q++=(unsigned char) (ScaleQuantumToShort(image->colormap[i].blue) & 0xff);
  packet_size=(size_t) (image->depth > 8 ? 2 : 1);
  status=SetQuantumDepth(image,quantum_info,image->depth > 8 ? 16 : 8);
    imbuf = (unsigned char *) AcquireQuantumMemory(imsx , imsy);
    (void) ResetMagickMemory(imbuf, background_color_index, (size_t) imsx * imsy);
                dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx , dmsy);
                (void) ResetMagickMemory(dmbuf, background_color_index, (size_t) dmsx * dmsy);
                dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx , dmsy);
                (void) ResetMagickMemory(dmbuf, background_color_index, (size_t) dmsx * dmsy);
                                (void) ResetMagickMemory(imbuf + (size_t) imsx * y + posision_x, color_index, repeat_count);
        if ((dmbuf = (unsigned char *) AcquireQuantumMemory(dmsx , dmsy)) == NULL) {
  (void) SetImageType(image,BilevelType,exception);