Commit message: https://github.com/ImageMagick/ImageMagick/issues/471
CVE: CVE-2017-11166
CWE: CWE-772
Vulnerability description: The ReadXWDImage function in coders\xwd.c in ImageMagick 7.0.5-6 has a memory leak vulnerability that can cause memory exhaustion via a crafted length (number of color-map entries) field in the header of an XWD file.

Project: ImageMagick
File: xwd.c

ModuleExport size_t RegisterXWDImage(void)
{
  MagickInfo
    *entry;

  entry=AcquireMagickInfo("XWD","XWD","X Windows system window dump (color)");
#if defined(MAGICKCORE_X11_DELEGATE)
  entry->decoder=(DecodeImageHandler *) ReadXWDImage;
  entry->encoder=(EncodeImageHandler *) WriteXWDImage;
#endif
  entry->magick=(IsImageFormatHandler *) IsXWD;
  entry->flags^=CoderAdjoinFlag;
  (void) RegisterMagickInfo(entry);
  return(MagickImageCoderSignature);
}

Project: ImageMagick
File: xwd.c

static Image *ReadXWDImage(const ImageInfo *image_info,ExceptionInfo *exception)
{
#define CheckOverflowException(length,width,height) \
  (((height) != 0) && ((length)/((size_t) height) != ((size_t) width)))

  char
    *comment;

  Image
    *image;

  int
    x_status;

  MagickBooleanType
    authentic_colormap;

  MagickStatusType
    status;

  Quantum
    index;

  register ssize_t
    x;

  register Quantum
    *q;

  register ssize_t
    i;

  register size_t
    pixel;

  size_t
    length;

  ssize_t
    count,
    y;

  unsigned long
    lsb_first;

  XColor
    *colors;

  XImage
    *ximage;

  XWDFileHeader
    header;

  /*
    Open image file.
  */
  assert(image_info != (const ImageInfo *) NULL);
  assert(image_info->signature == MagickCoreSignature);
  if (image_info->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",
      image_info->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  image=AcquireImage(image_info,exception);
  status=OpenBlob(image_info,image,ReadBinaryBlobMode,exception);
  if (status == MagickFalse)
    {
      image=DestroyImageList(image);
      return((Image *) NULL);
    }
  /*
     Read in header information.
  */
  count=ReadBlob(image,sz_XWDheader,(unsigned char *) &header);
  if (count != sz_XWDheader)
    ThrowReaderException(CorruptImageError,"UnableToReadImageHeader");
  /*
    Ensure the header byte-order is most-significant byte first.
  */
  lsb_first=1;
  if ((int) (*(char *) &lsb_first) != 0)
    MSBOrderLong((unsigned char *) &header,sz_XWDheader);
  /*
    Check to see if the dump file is in the proper format.
  */
  if (header.file_version != XWD_FILE_VERSION)
    ThrowReaderException(CorruptImageError,"FileFormatVersionMismatch");
  if (header.header_size < sz_XWDheader)
    ThrowReaderException(CorruptImageError,"CorruptImage");
  switch (header.visual_class) {
    case StaticGray:
    case GrayScale:
    case StaticColor:
    case PseudoColor:
    case TrueColor:
    case DirectColor:
      break;
    default:
      ThrowReaderException(CorruptImageError,"CorruptImage");
    }
  switch (header.pixmap_format) {
    case XYBitmap:
    case XYPixmap:
    case ZPixmap:
      break;
    default:
      ThrowReaderException(CorruptImageError,"CorruptImage");
  }
  length=(size_t) header.header_size-sz_XWDheader;
  comment=(char *) AcquireQuantumMemory(length+1,sizeof(*comment));
  if (comment == (char *) NULL)
    ThrowReaderException(ResourceLimitError,"MemoryAllocationFailed");
  count=ReadBlob(image,length,(unsigned char *) comment);
  comment[length]='\0';
  (void) SetImageProperty(image,"comment",comment,exception);
  comment=DestroyString(comment);
  if (count != (ssize_t) length)
    ThrowReaderException(CorruptImageError,"UnexpectedEndOfFile");
  /*
    Initialize the X image.
  */
  ximage=(XImage *) AcquireMagickMemory(sizeof(*ximage));
  if (ximage == (XImage *) NULL)
    ThrowReaderException(ResourceLimitError,"MemoryAllocationFailed");
  ximage->depth=(int) header.pixmap_depth;
  ximage->format=(int) header.pixmap_format;
  ximage->xoffset=(int) header.xoffset;
  ximage->data=(char *) NULL;
  ximage->width=(int) header.pixmap_width;
  ximage->height=(int) header.pixmap_height;
  ximage->bitmap_pad=(int) header.bitmap_pad;
  ximage->bytes_per_line=(int) header.bytes_per_line;
  ximage->byte_order=(int) header.byte_order;
  ximage->bitmap_unit=(int) header.bitmap_unit;
  ximage->bitmap_bit_order=(int) header.bitmap_bit_order;
  ximage->bits_per_pixel=(int) header.bits_per_pixel;
  ximage->red_mask=header.red_mask;
  ximage->green_mask=header.green_mask;
  ximage->blue_mask=header.blue_mask;
  if ((ximage->width < 0) || (ximage->height < 0) || (ximage->depth < 0) ||    
      (ximage->format < 0) || (ximage->byte_order < 0) || 
      (ximage->bitmap_bit_order < 0) || (ximage->bitmap_pad < 0) || 
      (ximage->bytes_per_line < 0))
    {
      ximage=(XImage *) RelinquishMagickMemory(ximage);
      ThrowReaderException(CorruptImageError,"ImproperImageHeader");
    }
  if ((ximage->width > 65535) || (ximage->height > 65535))
    {
      ximage=(XImage *) RelinquishMagickMemory(ximage);
      ThrowReaderException(CorruptImageError,"ImproperImageHeader");
    }
  if ((ximage->bits_per_pixel > 32) || (ximage->bitmap_unit > 32))
    {
      ximage=(XImage *) RelinquishMagickMemory(ximage);
      ThrowReaderException(CorruptImageError,"ImproperImageHeader");
    }
  x_status=XInitImage(ximage);
  if (x_status == 0)
    {
      ximage=(XImage *) RelinquishMagickMemory(ximage);
      ThrowReaderException(CorruptImageError,"UnexpectedEndOfFile");
    }
  /*
    Read colormap.
  */
  authentic_colormap=MagickFalse;
  colors=(XColor *) NULL;
  if (header.ncolors != 0)
    {
      XWDColor
        color;

      length=(size_t) header.ncolors;
      if (length > ((~0UL)/sizeof(*colors)))
        ThrowReaderException(CorruptImageError,"ImproperImageHeader");
      colors=(XColor *) AcquireQuantumMemory(length,sizeof(*colors));
      if (colors == (XColor *) NULL)
        {
          ximage=(XImage *) RelinquishMagickMemory(ximage);
          ThrowReaderException(ResourceLimitError,"MemoryAllocationFailed");
        }
      for (i=0; i < (ssize_t) header.ncolors; i++)
      {
        count=ReadBlob(image,sz_XWDColor,(unsigned char *) &color);
        if (count != sz_XWDColor)
          {
            colors=(XColor *) RelinquishMagickMemory(colors);
            ximage=(XImage *) RelinquishMagickMemory(ximage);
            ThrowReaderException(CorruptImageError,"UnexpectedEndOfFile");
          }
        colors[i].pixel=color.pixel;
        colors[i].red=color.red;
        colors[i].green=color.green;
        colors[i].blue=color.blue;
        colors[i].flags=(char) color.flags;
        if (color.flags != 0)
          authentic_colormap=MagickTrue;
      }
      /*
        Ensure the header byte-order is most-significant byte first.
      */
      lsb_first=1;
      if ((int) (*(char *) &lsb_first) != 0)
        for (i=0; i < (ssize_t) header.ncolors; i++)
        {
          MSBOrderLong((unsigned char *) &colors[i].pixel,
            sizeof(colors[i].pixel));
          MSBOrderShort((unsigned char *) &colors[i].red,3*
            sizeof(colors[i].red));
        }
    }
  /*
    Allocate the pixel buffer.
  */
  length=(size_t) ximage->bytes_per_line*ximage->height;
  if (CheckOverflowException(length,ximage->bytes_per_line,ximage->height))
    {
      if (header.ncolors != 0)
        colors=(XColor *) RelinquishMagickMemory(colors);
      ximage=(XImage *) RelinquishMagickMemory(ximage);
      ThrowReaderException(CorruptImageError,"ImproperImageHeader");
    }
  if (ximage->format != ZPixmap)
    {
      size_t
        extent;

      extent=length;
      length*=ximage->depth;
      if (CheckOverflowException(length,extent,ximage->depth))
        {
          if (header.ncolors != 0)
            colors=(XColor *) RelinquishMagickMemory(colors);
          ximage=(XImage *) RelinquishMagickMemory(ximage);
          ThrowReaderException(CorruptImageError,"ImproperImageHeader");
        }
    }
  ximage->data=(char *) AcquireQuantumMemory(length,sizeof(*ximage->data));
  if (ximage->data == (char *) NULL)
    {
      if (header.ncolors != 0)
        colors=(XColor *) RelinquishMagickMemory(colors);
      ximage=(XImage *) RelinquishMagickMemory(ximage);
      ThrowReaderException(ResourceLimitError,"MemoryAllocationFailed");
    }
  count=ReadBlob(image,length,(unsigned char *) ximage->data);
  if (count != (ssize_t) length)
    {
      if (header.ncolors != 0)
        colors=(XColor *) RelinquishMagickMemory(colors);
      ximage->data=DestroyString(ximage->data);
      ximage=(XImage *) RelinquishMagickMemory(ximage);
      ThrowReaderException(CorruptImageError,"UnableToReadImageData");
    }
  /*
    Convert image to MIFF format.
  */
  image->columns=(size_t) ximage->width;
  image->rows=(size_t) ximage->height;
  image->depth=8;
  status=SetImageExtent(image,image->columns,image->rows,exception);
  if (status == MagickFalse)
    {
      if (header.ncolors != 0)
        colors=(XColor *) RelinquishMagickMemory(colors);
      ximage->data=DestroyString(ximage->data);
      ximage=(XImage *) RelinquishMagickMemory(ximage);
      return(DestroyImageList(image));
    }
  if ((header.ncolors == 0U) || (ximage->red_mask != 0) ||
      (ximage->green_mask != 0) || (ximage->blue_mask != 0))
    image->storage_class=DirectClass;
  else
    image->storage_class=PseudoClass;
  image->colors=header.ncolors;
  if (image_info->ping == MagickFalse)
    switch (image->storage_class)
    {
      case DirectClass:
      default:
      {
        register size_t
          color;

        size_t
          blue_mask,
          blue_shift,
          green_mask,
          green_shift,
          red_mask,
          red_shift;

        /*
          Determine shift and mask for red, green, and blue.
        */
        red_mask=ximage->red_mask;
        red_shift=0;
        while ((red_mask != 0) && ((red_mask & 0x01) == 0))
        {
          red_mask>>=1;
          red_shift++;
        }
        green_mask=ximage->green_mask;
        green_shift=0;
        while ((green_mask != 0) && ((green_mask & 0x01) == 0))
        {
          green_mask>>=1;
          green_shift++;
        }
        blue_mask=ximage->blue_mask;
        blue_shift=0;
        while ((blue_mask != 0) && ((blue_mask & 0x01) == 0))
        {
          blue_mask>>=1;
          blue_shift++;
        }
        /*
          Convert X image to DirectClass packets.
        */
        if ((image->colors != 0) && (authentic_colormap != MagickFalse))
          for (y=0; y < (ssize_t) image->rows; y++)
          {
            q=QueueAuthenticPixels(image,0,y,image->columns,1,exception);
            if (q == (Quantum *) NULL)
              break;
            for (x=0; x < (ssize_t) image->columns; x++)
            {
              pixel=XGetPixel(ximage,(int) x,(int) y);
              index=(Quantum) ((pixel >> red_shift) & red_mask);
              if (index < header.ncolors)
                SetPixelRed(image,ScaleShortToQuantum(
                  colors[(ssize_t) index].red),q);
              index=(Quantum) ((pixel >> green_shift) & green_mask);
              if (index < header.ncolors)
                SetPixelGreen(image,ScaleShortToQuantum(
                  colors[(ssize_t) index].green),q);
              index=(Quantum) ((pixel >> blue_shift) & blue_mask);
              if (index < header.ncolors)
                SetPixelBlue(image,ScaleShortToQuantum(
                  colors[(ssize_t) index].blue),q);
              q+=GetPixelChannels(image);
            }
            if (SyncAuthenticPixels(image,exception) == MagickFalse)
              break;
            status=SetImageProgress(image,LoadImageTag,(MagickOffsetType) y,
              image->rows);
            if (status == MagickFalse)
              break;
          }
        else
          for (y=0; y < (ssize_t) image->rows; y++)
          {
            q=QueueAuthenticPixels(image,0,y,image->columns,1,exception);
            if (q == (Quantum *) NULL)
              break;
            for (x=0; x < (ssize_t) image->columns; x++)
            {
              pixel=XGetPixel(ximage,(int) x,(int) y);
              color=(pixel >> red_shift) & red_mask;
              if (red_mask != 0)
                color=(color*65535UL)/red_mask;
              SetPixelRed(image,ScaleShortToQuantum((unsigned short) color),q);
              color=(pixel >> green_shift) & green_mask;
              if (green_mask != 0)
                color=(color*65535UL)/green_mask;
              SetPixelGreen(image,ScaleShortToQuantum((unsigned short) color),
                q);
              color=(pixel >> blue_shift) & blue_mask;
              if (blue_mask != 0)
                color=(color*65535UL)/blue_mask;
              SetPixelBlue(image,ScaleShortToQuantum((unsigned short) color),q);
              q+=GetPixelChannels(image);
            }
            if (SyncAuthenticPixels(image,exception) == MagickFalse)
              break;
            status=SetImageProgress(image,LoadImageTag,(MagickOffsetType) y,
              image->rows);
            if (status == MagickFalse)
              break;
          }
        break;
      }
      case PseudoClass:
      {
        /*
          Convert X image to PseudoClass packets.
        */
        if (AcquireImageColormap(image,image->colors,exception) == MagickFalse)
          {
            if (header.ncolors != 0)
              colors=(XColor *) RelinquishMagickMemory(colors);
            ximage->data=DestroyString(ximage->data);
            ximage=(XImage *) RelinquishMagickMemory(ximage);
            ThrowReaderException(ResourceLimitError,"MemoryAllocationFailed");
          }
        for (i=0; i < (ssize_t) image->colors; i++)
        {
          image->colormap[i].red=(MagickRealType) ScaleShortToQuantum(
            colors[i].red);
          image->colormap[i].green=(MagickRealType) ScaleShortToQuantum(
            colors[i].green);
          image->colormap[i].blue=(MagickRealType) ScaleShortToQuantum(
            colors[i].blue);
        }
        for (y=0; y < (ssize_t) image->rows; y++)
        {
          q=QueueAuthenticPixels(image,0,y,image->columns,1,exception);
          if (q == (Quantum *) NULL)
            break;
          for (x=0; x < (ssize_t) image->columns; x++)
          {
            index=ConstrainColormapIndex(image,XGetPixel(ximage,(int) x,
              (int) y),exception);
            SetPixelIndex(image,index,q);
            SetPixelViaPixelInfo(image,image->colormap+(ssize_t) index,q);
            q+=GetPixelChannels(image);
          }
          if (SyncAuthenticPixels(image,exception) == MagickFalse)
            break;
          status=SetImageProgress(image,LoadImageTag,(MagickOffsetType) y,
            image->rows);
          if (status == MagickFalse)
            break;
        }
        break;
      }
    }
  /*
    Free image and colormap.
  */
  if (header.ncolors != 0)
    colors=(XColor *) RelinquishMagickMemory(colors);
  ximage->data=DestroyString(ximage->data);
  ximage=(XImage *) RelinquishMagickMemory(ximage);
  if (EOFBlob(image) != MagickFalse)
    ThrowFileException(exception,CorruptImageError,"UnexpectedEndOfFile",
      image->filename);
  (void) CloseBlob(image);
  return(GetFirstImageInList(image));
}

Project: ImageMagick
File: xwd.c

static MagickBooleanType WriteXWDImage(const ImageInfo *image_info,Image *image,
  ExceptionInfo *exception)
{
  const char
    *value;

  MagickBooleanType
    status;

  register const Quantum
    *p;

  register ssize_t
    x;

  register unsigned char
    *q;

  size_t
    bits_per_pixel,
    bytes_per_line,
    length,
    scanline_pad;

  ssize_t
    y;

  unsigned char
    *pixels;

  unsigned long
    lsb_first;

  XWDFileHeader
    xwd_info;

  /*
    Open output image file.
  */
  assert(image_info != (const ImageInfo *) NULL);
  assert(image_info->signature == MagickCoreSignature);
  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  status=OpenBlob(image_info,image,WriteBinaryBlobMode,exception);
  if (status == MagickFalse)
    return(status);
  (void) TransformImageColorspace(image,sRGBColorspace,exception);
  /*
    Initialize XWD file header.
  */
  (void) ResetMagickMemory(&xwd_info,0,sizeof(xwd_info));
  xwd_info.header_size=(CARD32) sz_XWDheader;
  value=GetImageProperty(image,"comment",exception);
  if (value != (const char *) NULL)
    xwd_info.header_size+=(CARD32) strlen(value);
  xwd_info.header_size++;
  xwd_info.file_version=(CARD32) XWD_FILE_VERSION;
  xwd_info.pixmap_format=(CARD32) ZPixmap;
  xwd_info.pixmap_depth=(CARD32) (image->storage_class == DirectClass ? 24 : 8);
  xwd_info.pixmap_width=(CARD32) image->columns;
  xwd_info.pixmap_height=(CARD32) image->rows;
  xwd_info.xoffset=(CARD32) 0;
  xwd_info.byte_order=(CARD32) MSBFirst;
  xwd_info.bitmap_unit=(CARD32) (image->storage_class == DirectClass ? 32 : 8);
  xwd_info.bitmap_bit_order=(CARD32) MSBFirst;
  xwd_info.bitmap_pad=(CARD32) (image->storage_class == DirectClass ? 32 : 8);
  bits_per_pixel=(size_t) (image->storage_class == DirectClass ? 24 : 8);
  xwd_info.bits_per_pixel=(CARD32) bits_per_pixel;
  bytes_per_line=(CARD32) ((((xwd_info.bits_per_pixel*
    xwd_info.pixmap_width)+((xwd_info.bitmap_pad)-1))/
    (xwd_info.bitmap_pad))*((xwd_info.bitmap_pad) >> 3));
  xwd_info.bytes_per_line=(CARD32) bytes_per_line;
  xwd_info.visual_class=(CARD32)
    (image->storage_class == DirectClass ? DirectColor : PseudoColor);
  xwd_info.red_mask=(CARD32)
    (image->storage_class == DirectClass ? 0xff0000 : 0);
  xwd_info.green_mask=(CARD32)
    (image->storage_class == DirectClass ? 0xff00 : 0);
  xwd_info.blue_mask=(CARD32) (image->storage_class == DirectClass ? 0xff : 0);
  xwd_info.bits_per_rgb=(CARD32) (image->storage_class == DirectClass ? 24 : 8);
  xwd_info.colormap_entries=(CARD32)
    (image->storage_class == DirectClass ? 256 : image->colors);
  xwd_info.ncolors=(unsigned int)
    (image->storage_class == DirectClass ? 0 : image->colors);
  xwd_info.window_width=(CARD32) image->columns;
  xwd_info.window_height=(CARD32) image->rows;
  xwd_info.window_x=0;
  xwd_info.window_y=0;
  xwd_info.window_bdrwidth=(CARD32) 0;
  /*
    Write XWD header.
  */
  lsb_first=1;
  if ((int) (*(char *) &lsb_first) != 0)
    MSBOrderLong((unsigned char *) &xwd_info,sizeof(xwd_info));
  (void) WriteBlob(image,sz_XWDheader,(unsigned char *) &xwd_info);
  if (value != (const char *) NULL)
    (void) WriteBlob(image,strlen(value),(unsigned char *) value);
  (void) WriteBlob(image,1,(const unsigned char *) "\0");
  if (image->storage_class == PseudoClass)
    {
      register ssize_t
        i;

      XColor
        *colors;

      XWDColor
        color;

      /*
        Dump colormap to file.
      */
      (void) ResetMagickMemory(&color,0,sizeof(color));
      colors=(XColor *) AcquireQuantumMemory((size_t) image->colors,
        sizeof(*colors));
      if (colors == (XColor *) NULL)
        ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
      for (i=0; i < (ssize_t) image->colors; i++)
      {
        colors[i].pixel=(unsigned long) i;
        colors[i].red=ScaleQuantumToShort(ClampToQuantum(
          image->colormap[i].red));
        colors[i].green=ScaleQuantumToShort(ClampToQuantum(
          image->colormap[i].green));
        colors[i].blue=ScaleQuantumToShort(ClampToQuantum(
          image->colormap[i].blue));
        colors[i].flags=(char) (DoRed | DoGreen | DoBlue);
        colors[i].pad='\0';
        if ((int) (*(char *) &lsb_first) != 0)
          {
            MSBOrderLong((unsigned char *) &colors[i].pixel,
              sizeof(colors[i].pixel));
            MSBOrderShort((unsigned char *) &colors[i].red,
              3*sizeof(colors[i].red));
          }
      }
      for (i=0; i < (ssize_t) image->colors; i++)
      {
        color.pixel=(CARD32) colors[i].pixel;
        color.red=colors[i].red;
        color.green=colors[i].green;
        color.blue=colors[i].blue;
        color.flags=(CARD8) colors[i].flags;
        (void) WriteBlob(image,sz_XWDColor,(unsigned char *) &color);
      }
      colors=(XColor *) RelinquishMagickMemory(colors);
    }
  /*
    Allocate memory for pixels.
  */
  length=3*bytes_per_line;
  if (image->storage_class == PseudoClass)
    length=bytes_per_line;
  pixels=(unsigned char *) AcquireQuantumMemory(length,sizeof(*pixels));
  if (pixels == (unsigned char *) NULL)
    ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
  (void) ResetMagickMemory(pixels,0,length);
  /*
    Convert MIFF to XWD raster pixels.
  */
  scanline_pad=(bytes_per_line-((image->columns*bits_per_pixel) >> 3));
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    p=GetVirtualPixels(image,0,y,image->columns,1,exception);
    if (p == (const Quantum *) NULL)
      break;
    q=pixels;
    if (image->storage_class == PseudoClass)
      {
        for (x=0; x < (ssize_t) image->columns; x++)
        {
          *q++=(unsigned char) GetPixelIndex(image,p);
          p+=GetPixelChannels(image);
        }
      }
    else
      for (x=0; x < (ssize_t) image->columns; x++)
      {
        *q++=ScaleQuantumToChar(GetPixelRed(image,p));
        *q++=ScaleQuantumToChar(GetPixelGreen(image,p));
        *q++=ScaleQuantumToChar(GetPixelBlue(image,p));
        p+=GetPixelChannels(image);
      }
    for (x=0; x < (ssize_t) scanline_pad; x++)
      *q++='\0';
    (void) WriteBlob(image,(size_t) (q-pixels),pixels);
    status=SetImageProgress(image,SaveImageTag,(MagickOffsetType) y,
      image->rows);
    if (status == MagickFalse)
      break;
  }
  pixels=(unsigned char *) RelinquishMagickMemory(pixels);
  (void) CloseBlob(image);
  return(MagickTrue);
}