Commit message: https://github.com/ImageMagick/ImageMagick/issues/1540
CVE: CVE-2019-11598
CWE: CWE-125
Vulnerability description: In ImageMagick 7.0.8-40 Q16, there is a heap-based buffer over-read in the function WritePNMImage of coders/pnm.c, which allows an attacker to cause a denial of service or possibly information disclosure via a crafted image file. This is related to SetGrayscaleImage in MagickCore/quantize.c.

Project: ImageMagick6

MagickExport MagickBooleanType GetImageQuantizeError(Image *image)
{
  CacheView
    *image_view;

  ExceptionInfo
    *exception;

  IndexPacket
    *indexes;

  MagickRealType
    alpha,
    area,
    beta,
    distance,
    gamma,
    maximum_error,
    mean_error,
    mean_error_per_pixel;

  size_t
    index;

  ssize_t
    y;

  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  image->total_colors=GetNumberColors(image,(FILE *) NULL,&image->exception);
  (void) memset(&image->error,0,sizeof(image->error));
  if (image->storage_class == DirectClass)
    return(MagickTrue);
  alpha=1.0;
  beta=1.0;
  area=3.0*image->columns*image->rows;
  maximum_error=0.0;
  mean_error_per_pixel=0.0;
  mean_error=0.0;
  exception=(&image->exception);
  image_view=AcquireVirtualCacheView(image,exception);
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    register const PixelPacket
      *magick_restrict p;

    register ssize_t
      x;

    p=GetCacheViewVirtualPixels(image_view,0,y,image->columns,1,exception);
    if (p == (const PixelPacket *) NULL)
      break;
    indexes=GetCacheViewAuthenticIndexQueue(image_view);
    for (x=0; x < (ssize_t) image->columns; x++)
    {
      index=1UL*GetPixelIndex(indexes+x);
      if (image->matte != MagickFalse)
        {
          alpha=(MagickRealType) (QuantumScale*(GetPixelAlpha(p)));
          beta=(MagickRealType) (QuantumScale*(QuantumRange-
            image->colormap[index].opacity));
        }
      distance=fabs((double) (alpha*GetPixelRed(p)-beta*
        image->colormap[index].red));
      mean_error_per_pixel+=distance;
      mean_error+=distance*distance;
      if (distance > maximum_error)
        maximum_error=distance;
      distance=fabs((double) (alpha*GetPixelGreen(p)-beta*
        image->colormap[index].green));
      mean_error_per_pixel+=distance;
      mean_error+=distance*distance;
      if (distance > maximum_error)
        maximum_error=distance;
      distance=fabs((double) (alpha*GetPixelBlue(p)-beta*
        image->colormap[index].blue));
      mean_error_per_pixel+=distance;
      mean_error+=distance*distance;
      if (distance > maximum_error)
        maximum_error=distance;
      p++;
    }
  }
  image_view=DestroyCacheView(image_view);
  gamma=PerceptibleReciprocal(area);
  image->error.mean_error_per_pixel=gamma*mean_error_per_pixel;
  image->error.normalized_mean_error=gamma*QuantumScale*QuantumScale*mean_error;
  image->error.normalized_maximum_error=QuantumScale*maximum_error;
  return(MagickTrue);
}

Project: ImageMagick6

MagickExport MagickBooleanType PosterizeImageChannel(Image *image,
  const ChannelType channel,const size_t levels,const MagickBooleanType dither)
{
#define PosterizeImageTag  "Posterize/Image"
#define PosterizePixel(pixel) (Quantum) (QuantumRange*(MagickRound( \
  QuantumScale*pixel*(levels-1)))/MagickMax((ssize_t) levels-1,1))

  CacheView
    *image_view;

  ExceptionInfo
    *exception;

  MagickBooleanType
    status;

  MagickOffsetType
    progress;

  QuantizeInfo
    *quantize_info;

  register ssize_t
    i;

  ssize_t
    y;

  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  if (image->storage_class == PseudoClass)
#if defined(MAGICKCORE_OPENMP_SUPPORT)
    #pragma omp parallel for schedule(static) shared(progress,status) \
      magick_number_threads(image,image,image->colors,1)
#endif
    for (i=0; i < (ssize_t) image->colors; i++)
    {
      /*
        Posterize colormap.
      */
      if ((channel & RedChannel) != 0)
        image->colormap[i].red=PosterizePixel(image->colormap[i].red);
      if ((channel & GreenChannel) != 0)
        image->colormap[i].green=PosterizePixel(image->colormap[i].green);
      if ((channel & BlueChannel) != 0)
        image->colormap[i].blue=PosterizePixel(image->colormap[i].blue);
      if ((channel & OpacityChannel) != 0)
        image->colormap[i].opacity=PosterizePixel(image->colormap[i].opacity);
    }
  /*
    Posterize image.
  */
  status=MagickTrue;
  progress=0;
  exception=(&image->exception);
  image_view=AcquireAuthenticCacheView(image,exception);
#if defined(MAGICKCORE_OPENMP_SUPPORT)
  #pragma omp parallel for schedule(static) shared(progress,status) \
    magick_number_threads(image,image,image->rows,1)
#endif
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    register IndexPacket
      *magick_restrict indexes;

    register PixelPacket
      *magick_restrict q;

    register ssize_t
      x;

    if (status == MagickFalse)
      continue;
    q=GetCacheViewAuthenticPixels(image_view,0,y,image->columns,1,exception);
    if (q == (PixelPacket *) NULL)
      {
        status=MagickFalse;
        continue;
      }
    indexes=GetCacheViewAuthenticIndexQueue(image_view);
    for (x=0; x < (ssize_t) image->columns; x++)
    {
      if ((channel & RedChannel) != 0)
        SetPixelRed(q,PosterizePixel(GetPixelRed(q)));
      if ((channel & GreenChannel) != 0)
        SetPixelGreen(q,PosterizePixel(GetPixelGreen(q)));
      if ((channel & BlueChannel) != 0)
        SetPixelBlue(q,PosterizePixel(GetPixelBlue(q)));
      if (((channel & OpacityChannel) != 0) &&
          (image->matte != MagickFalse))
        SetPixelOpacity(q,PosterizePixel(GetPixelOpacity(q)));
      if (((channel & IndexChannel) != 0) &&
          (image->colorspace == CMYKColorspace))
        SetPixelIndex(indexes+x,PosterizePixel(GetPixelIndex(indexes+x)));
      q++;
    }
    if (SyncCacheViewAuthenticPixels(image_view,exception) == MagickFalse)
      status=MagickFalse;
    if (image->progress_monitor != (MagickProgressMonitor) NULL)
      {
        MagickBooleanType
          proceed;

#if defined(MAGICKCORE_OPENMP_SUPPORT)
        #pragma omp atomic
#endif
        progress++;
        proceed=SetImageProgress(image,PosterizeImageTag,progress,image->rows);
        if (proceed == MagickFalse)
          status=MagickFalse;
      }
  }
  image_view=DestroyCacheView(image_view);
  quantize_info=AcquireQuantizeInfo((ImageInfo *) NULL);
  quantize_info->number_colors=(size_t) MagickMin((ssize_t) levels*levels*
    levels,MaxColormapSize+1);
  quantize_info->dither=dither;
  quantize_info->tree_depth=MaxTreeDepth;
  status=QuantizeImage(quantize_info,image);
  quantize_info=DestroyQuantizeInfo(quantize_info);
  return(status);
}

Project: ImageMagick6

static MagickBooleanType SetGrayscaleImage(Image *image)
{
  CacheView
    *image_view;

  ExceptionInfo
    *exception;

  MagickBooleanType
    status;

  PixelPacket
    *colormap;

  register ssize_t
    i;

  ssize_t
    *colormap_index,
    j,
    y;

  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  exception=(&image->exception);
  if (image->type != GrayscaleType)
    (void) TransformImageColorspace(image,GRAYColorspace);
  if (image->storage_class == PseudoClass)
    colormap_index=(ssize_t *) AcquireQuantumMemory(MagickMax(image->colors+1,
      MaxMap),sizeof(*colormap_index));
  else
    colormap_index=(ssize_t *) AcquireQuantumMemory(MagickMax(MaxColormapSize+1,
      MaxMap),sizeof(*colormap_index));
  if (colormap_index == (ssize_t *) NULL)
    ThrowBinaryException(ResourceLimitError,"MemoryAllocationFailed",
      image->filename);
  if (image->storage_class != PseudoClass)
    {
      (void) memset(colormap_index,(-1),MaxColormapSize*
        sizeof(*colormap_index));
      if (AcquireImageColormap(image,MaxColormapSize) == MagickFalse)
        {
          colormap_index=(ssize_t *) RelinquishMagickMemory(colormap_index);
          ThrowBinaryException(ResourceLimitError,"MemoryAllocationFailed",
            image->filename);
        }
      image->colors=0;
      status=MagickTrue;
      image_view=AcquireAuthenticCacheView(image,exception);
#if defined(MAGICKCORE_OPENMP_SUPPORT)
      #pragma omp parallel for schedule(static) shared(status) \
        magick_number_threads(image,image,image->rows,1)
#endif
      for (y=0; y < (ssize_t) image->rows; y++)
      {
        register IndexPacket
          *magick_restrict indexes;

        register PixelPacket
          *magick_restrict q;

        register ssize_t
          x;

        if (status == MagickFalse)
          continue;
        q=GetCacheViewAuthenticPixels(image_view,0,y,image->columns,1,
          exception);
        if (q == (PixelPacket *) NULL)
          {
            status=MagickFalse;
            continue;
          }
        indexes=GetCacheViewAuthenticIndexQueue(image_view);
        for (x=0; x < (ssize_t) image->columns; x++)
        {
          register size_t
            intensity;

          intensity=ScaleQuantumToMap(GetPixelRed(q));
          if (colormap_index[intensity] < 0)
            {
#if defined(MAGICKCORE_OPENMP_SUPPORT)
              #pragma omp critical (MagickCore_SetGrayscaleImage)
#endif
              if (colormap_index[intensity] < 0)
                {
                  colormap_index[intensity]=(ssize_t) image->colors;
                  image->colormap[image->colors].red=GetPixelRed(q);
                  image->colormap[image->colors].green=GetPixelGreen(q);
                  image->colormap[image->colors].blue=GetPixelBlue(q);
                  image->colors++;
               }
            }
          SetPixelIndex(indexes+x,colormap_index[intensity]);
          q++;
        }
        if (SyncCacheViewAuthenticPixels(image_view,exception) == MagickFalse)
          status=MagickFalse;
      }
      image_view=DestroyCacheView(image_view);
    }
  for (i=0; i < (ssize_t) image->colors; i++)
    image->colormap[i].opacity=(unsigned short) i;
  qsort((void *) image->colormap,image->colors,sizeof(PixelPacket),
    IntensityCompare);
  colormap=(PixelPacket *) AcquireQuantumMemory(image->colors,
    sizeof(*colormap));
  if (colormap == (PixelPacket *) NULL)
    {
      colormap_index=(ssize_t *) RelinquishMagickMemory(colormap_index);
      ThrowBinaryException(ResourceLimitError,"MemoryAllocationFailed",
        image->filename);
    }
  j=0;
  colormap[j]=image->colormap[0];
  for (i=0; i < (ssize_t) image->colors; i++)
  {
    if (IsSameColor(image,&colormap[j],&image->colormap[i]) == MagickFalse)
      {
        j++;
        colormap[j]=image->colormap[i];
      }
    colormap_index[(ssize_t) image->colormap[i].opacity]=j;
  }
  image->colors=(size_t) (j+1);
  image->colormap=(PixelPacket *) RelinquishMagickMemory(image->colormap);
  image->colormap=colormap;
  status=MagickTrue;
  image_view=AcquireAuthenticCacheView(image,exception);
#if defined(MAGICKCORE_OPENMP_SUPPORT)
  #pragma omp parallel for schedule(static) shared(status) \
    magick_number_threads(image,image,image->rows,1)
#endif
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    register IndexPacket
      *magick_restrict indexes;

    register const PixelPacket
      *magick_restrict q;

    register ssize_t
      x;

    if (status == MagickFalse)
      continue;
    q=GetCacheViewAuthenticPixels(image_view,0,y,image->columns,1,exception);
    if (q == (PixelPacket *) NULL)
      {
        status=MagickFalse;
        continue;
      }
    indexes=GetCacheViewAuthenticIndexQueue(image_view);
    for (x=0; x < (ssize_t) image->columns; x++)
      SetPixelIndex(indexes+x,colormap_index[ScaleQuantumToMap(GetPixelIndex(
        indexes+x))]);
    if (SyncCacheViewAuthenticPixels(image_view,exception) == MagickFalse)
      status=MagickFalse;
  }
  image_view=DestroyCacheView(image_view);
  colormap_index=(ssize_t *) RelinquishMagickMemory(colormap_index);
  image->type=GrayscaleType;
  if (SetImageMonochrome(image,&image->exception) != MagickFalse)
    image->type=BilevelType;
  return(status);
}

Project: ImageMagick6

MagickExport MagickBooleanType QuantizeImage(const QuantizeInfo *quantize_info,
  Image *image)
{
  CubeInfo
    *cube_info;

  MagickBooleanType
    status;

  size_t
    depth,
    maximum_colors;

  assert(quantize_info != (const QuantizeInfo *) NULL);
  assert(quantize_info->signature == MagickCoreSignature);
  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  maximum_colors=quantize_info->number_colors;
  if (maximum_colors == 0)
    maximum_colors=MaxColormapSize;
  if (maximum_colors > MaxColormapSize)
    maximum_colors=MaxColormapSize;
  if (image->matte == MagickFalse)
    {
      if (SetImageGray(image,&image->exception) != MagickFalse)
        (void) SetGrayscaleImage(image);
    }
  if ((image->storage_class == PseudoClass) &&
      (image->colors <= maximum_colors))
    {
      if ((quantize_info->colorspace != UndefinedColorspace) &&
          (quantize_info->colorspace != CMYKColorspace))
        (void) TransformImageColorspace(image,quantize_info->colorspace);
      return(MagickTrue);
    }
  depth=quantize_info->tree_depth;
  if (depth == 0)
    {
      size_t
        colors;

      /*
        Depth of color tree is: Log4(colormap size)+2.
      */
      colors=maximum_colors;
      for (depth=1; colors != 0; depth++)
        colors>>=2;
      if ((quantize_info->dither != MagickFalse) && (depth > 2))
        depth--;
      if ((image->matte != MagickFalse) && (depth > 5))
        depth--;
      if (SetImageGray(image,&image->exception) != MagickFalse)
        depth=MaxTreeDepth;
    }
  /*
    Initialize color cube.
  */
  cube_info=GetCubeInfo(quantize_info,depth,maximum_colors);
  if (cube_info == (CubeInfo *) NULL)
    ThrowBinaryImageException(ResourceLimitError,"MemoryAllocationFailed",
      image->filename);
  status=ClassifyImageColors(cube_info,image,&image->exception);
  if (status != MagickFalse)
    {
      /*
        Reduce the number of colors in the image if it contains more than the
        maximum, otherwise we can disable dithering to improve the performance.
      */
      if (cube_info->colors > cube_info->maximum_colors)
        ReduceImageColors(image,cube_info);
      else
        cube_info->quantize_info->dither_method=NoDitherMethod;
      status=AssignImageColors(image,cube_info);
    }
  DestroyCubeInfo(cube_info);
  return(status);
}