Commit message: https://github.com/ImageMagick/ImageMagick/issues/1540
CVE: CVE-2019-11598
CWE: CWE-125
Vulnerability description: In ImageMagick 7.0.8-40 Q16, there is a heap-based buffer over-read in the function WritePNMImage of coders/pnm.c, which allows an attacker to cause a denial of service or possibly information disclosure via a crafted image file. This is related to SetGrayscaleImage in MagickCore/quantize.c.

Project: ImageMagick
File: quantize.c

static MagickBooleanType SetGrayscaleImage(Image *image,
  ExceptionInfo *exception)
{
  CacheView
    *image_view;

  MagickBooleanType
    status;

  PixelInfo
    *colormap;

  register ssize_t
    i;

  ssize_t
    *colormap_index,
    j,
    y;

  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->type != GrayscaleType)
    (void) TransformImageColorspace(image,GRAYColorspace,exception);
  if (image->storage_class == PseudoClass)
    colormap_index=(ssize_t *) AcquireQuantumMemory(MagickMax(image->colors+1,
      MaxMap),sizeof(*colormap_index));
  else
    colormap_index=(ssize_t *) AcquireQuantumMemory(MagickMax(MaxColormapSize+1,
      MaxMap),sizeof(*colormap_index));
  if (colormap_index == (ssize_t *) NULL)
    ThrowBinaryException(ResourceLimitError,"MemoryAllocationFailed",
      image->filename);
  if (image->storage_class != PseudoClass)
    {
      (void) memset(colormap_index,(-1),MaxColormapSize*
        sizeof(*colormap_index));
      if (AcquireImageColormap(image,MaxColormapSize,exception) == MagickFalse)
        {
          colormap_index=(ssize_t *) RelinquishMagickMemory(colormap_index);
          ThrowBinaryException(ResourceLimitError,"MemoryAllocationFailed",
            image->filename);
        }
      image->colors=0;
      status=MagickTrue;
      image_view=AcquireAuthenticCacheView(image,exception);
#if defined(MAGICKCORE_OPENMP_SUPPORT)
      #pragma omp parallel for schedule(static) shared(status) \
        magick_number_threads(image,image,image->rows,1)
#endif
      for (y=0; y < (ssize_t) image->rows; y++)
      {
        register Quantum
          *magick_restrict q;

        register ssize_t
          x;

        if (status == MagickFalse)
          continue;
        q=GetCacheViewAuthenticPixels(image_view,0,y,image->columns,1,
          exception);
        if (q == (Quantum *) NULL)
          {
            status=MagickFalse;
            continue;
          }
        for (x=0; x < (ssize_t) image->columns; x++)
        {
          register size_t
            intensity;

          intensity=ScaleQuantumToMap(GetPixelRed(image,q));
          if (colormap_index[intensity] < 0)
            {
#if defined(MAGICKCORE_OPENMP_SUPPORT)
              #pragma omp critical (MagickCore_SetGrayscaleImage)
#endif
              if (colormap_index[intensity] < 0)
                {
                  colormap_index[intensity]=(ssize_t) image->colors;
                  image->colormap[image->colors].red=(double)
                    GetPixelRed(image,q);
                  image->colormap[image->colors].green=(double)
                    GetPixelGreen(image,q);
                  image->colormap[image->colors].blue=(double)
                    GetPixelBlue(image,q);
                  image->colors++;
               }
            }
          SetPixelIndex(image,(Quantum) colormap_index[intensity],q);
          q+=GetPixelChannels(image);
        }
        if (SyncCacheViewAuthenticPixels(image_view,exception) == MagickFalse)
          status=MagickFalse;
      }
      image_view=DestroyCacheView(image_view);
    }
  for (i=0; i < (ssize_t) image->colors; i++)
    image->colormap[i].alpha=(double) i;
  qsort((void *) image->colormap,image->colors,sizeof(PixelInfo),
    IntensityCompare);
  colormap=(PixelInfo *) AcquireQuantumMemory(image->colors,sizeof(*colormap));
  if (colormap == (PixelInfo *) NULL)
    {
      colormap_index=(ssize_t *) RelinquishMagickMemory(colormap_index);
      ThrowBinaryException(ResourceLimitError,"MemoryAllocationFailed",
        image->filename);
    }
  j=0;
  colormap[j]=image->colormap[0];
  for (i=0; i < (ssize_t) image->colors; i++)
  {
    if (IsPixelInfoEquivalent(&colormap[j],&image->colormap[i]) == MagickFalse)
      {
        j++;
        colormap[j]=image->colormap[i];
      }
    colormap_index[(ssize_t) image->colormap[i].alpha]=j;
  }
  image->colors=(size_t) (j+1);
  image->colormap=(PixelInfo *) RelinquishMagickMemory(image->colormap);
  image->colormap=colormap;
  status=MagickTrue;
  image_view=AcquireAuthenticCacheView(image,exception);
#if defined(MAGICKCORE_OPENMP_SUPPORT)
  #pragma omp parallel for schedule(static) shared(status) \
    magick_number_threads(image,image,image->rows,1)
#endif
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    register Quantum
      *magick_restrict q;

    register ssize_t
      x;

    if (status == MagickFalse)
      continue;
    q=GetCacheViewAuthenticPixels(image_view,0,y,image->columns,1,exception);
    if (q == (Quantum *) NULL)
      {
        status=MagickFalse;
        continue;
      }
    for (x=0; x < (ssize_t) image->columns; x++)
    {
      SetPixelIndex(image,(Quantum) colormap_index[ScaleQuantumToMap(
        GetPixelIndex(image,q))],q);
      q+=GetPixelChannels(image);
    }
    if (SyncCacheViewAuthenticPixels(image_view,exception) == MagickFalse)
      status=MagickFalse;
  }
  image_view=DestroyCacheView(image_view);
  colormap_index=(ssize_t *) RelinquishMagickMemory(colormap_index);
  image->type=GrayscaleType;
  if (SetImageMonochrome(image,exception) != MagickFalse)
    image->type=BilevelType;
  return(status);
}

Project: ImageMagick
File: quantize.c

static MagickBooleanType AssignImageColors(Image *image,CubeInfo *cube_info,
  ExceptionInfo *exception)
{
#define AssignImageTag  "Assign/Image"

  ColorspaceType
    colorspace;

  ssize_t
    y;

  /*
    Allocate image colormap.
  */
  colorspace=image->colorspace;
  if (cube_info->quantize_info->colorspace != UndefinedColorspace)
    (void) TransformImageColorspace(image,cube_info->quantize_info->colorspace,
      exception);
  if (AcquireImageColormap(image,cube_info->colors,exception) == MagickFalse)
    ThrowBinaryException(ResourceLimitError,"MemoryAllocationFailed",
      image->filename);
  image->colors=0;
  cube_info->transparent_pixels=0;
  cube_info->transparent_index=(-1);
  (void) DefineImageColormap(image,cube_info,cube_info->root);
  /*
    Create a reduced color image.
  */
  if (cube_info->quantize_info->dither_method != NoDitherMethod)
    (void) DitherImage(image,cube_info,exception);
  else
    {
      CacheView
        *image_view;

      MagickBooleanType
        status;

      status=MagickTrue;
      image_view=AcquireAuthenticCacheView(image,exception);
#if defined(MAGICKCORE_OPENMP_SUPPORT)
      #pragma omp parallel for schedule(static) shared(status) \
        magick_number_threads(image,image,image->rows,1)
#endif
      for (y=0; y < (ssize_t) image->rows; y++)
      {
        CubeInfo
          cube;

        register Quantum
          *magick_restrict q;

        register ssize_t
          x;

        ssize_t
          count;

        if (status == MagickFalse)
          continue;
        q=GetCacheViewAuthenticPixels(image_view,0,y,image->columns,1,
          exception);
        if (q == (Quantum *) NULL)
          {
            status=MagickFalse;
            continue;
          }
        cube=(*cube_info);
        for (x=0; x < (ssize_t) image->columns; x+=count)
        {
          DoublePixelPacket
            pixel;

          register const NodeInfo
            *node_info;

          register ssize_t
            i;

          size_t
            id,
            index;

          /*
            Identify the deepest node containing the pixel's color.
          */
          for (count=1; (x+count) < (ssize_t) image->columns; count++)
          {
            PixelInfo
              packet;

            GetPixelInfoPixel(image,q+count*GetPixelChannels(image),&packet);
            if (IsPixelEquivalent(image,q,&packet) == MagickFalse)
              break;
          }
          AssociateAlphaPixel(image,&cube,q,&pixel);
          node_info=cube.root;
          for (index=MaxTreeDepth-1; (ssize_t) index > 0; index--)
          {
            id=ColorToNodeId(&cube,&pixel,index);
            if (node_info->child[id] == (NodeInfo *) NULL)
              break;
            node_info=node_info->child[id];
          }
          /*
            Find closest color among siblings and their children.
          */
          cube.target=pixel;
          cube.distance=(double) (4.0*(QuantumRange+1.0)*(QuantumRange+1.0)+
            1.0);
          ClosestColor(image,&cube,node_info->parent);
          index=cube.color_number;
          for (i=0; i < (ssize_t) count; i++)
          {
            if (image->storage_class == PseudoClass)
              SetPixelIndex(image,(Quantum) index,q);
            if (cube.quantize_info->measure_error == MagickFalse)
              {
                SetPixelRed(image,ClampToQuantum(
                  image->colormap[index].red),q);
                SetPixelGreen(image,ClampToQuantum(
                  image->colormap[index].green),q);
                SetPixelBlue(image,ClampToQuantum(
                  image->colormap[index].blue),q);
                if (cube.associate_alpha != MagickFalse)
                  SetPixelAlpha(image,ClampToQuantum(
                    image->colormap[index].alpha),q);
              }
            q+=GetPixelChannels(image);
          }
        }
        if (SyncCacheViewAuthenticPixels(image_view,exception) == MagickFalse)
          status=MagickFalse;
        if (image->progress_monitor != (MagickProgressMonitor) NULL)
          {
            MagickBooleanType
              proceed;

            proceed=SetImageProgress(image,AssignImageTag,(MagickOffsetType) y,
              image->rows);
            if (proceed == MagickFalse)
              status=MagickFalse;
          }
      }
      image_view=DestroyCacheView(image_view);
    }
  if (cube_info->quantize_info->measure_error != MagickFalse)
    (void) GetImageQuantizeError(image,exception);
  if ((cube_info->quantize_info->number_colors == 2) &&
      ((cube_info->quantize_info->colorspace == LinearGRAYColorspace) ||
       (cube_info->quantize_info->colorspace == GRAYColorspace)))
    {
      double
        intensity;

      /*
        Monochrome image.
      */
      intensity=0.0;
      if ((image->colors > 1) &&
          (GetPixelInfoLuma(image->colormap+0) >
           GetPixelInfoLuma(image->colormap+1)))
        intensity=(double) QuantumRange;
      image->colormap[0].red=intensity;
      image->colormap[0].green=intensity;
      image->colormap[0].blue=intensity;
      if (image->colors > 1)
        {
          image->colormap[1].red=(double) QuantumRange-intensity;
          image->colormap[1].green=(double) QuantumRange-intensity;
          image->colormap[1].blue=(double) QuantumRange-intensity;
        }
    }
  (void) SyncImage(image,exception);
  if ((cube_info->quantize_info->colorspace != UndefinedColorspace) &&
      (IssRGBCompatibleColorspace(colorspace) == MagickFalse))
    (void) TransformImageColorspace(image,colorspace,exception);
  return(MagickTrue);
}

Project: ImageMagick
File: quantize.c

static MagickBooleanType FloydSteinbergDither(Image *image,CubeInfo *cube_info,
  ExceptionInfo *exception)
{
#define DitherImageTag  "Dither/Image"

  CacheView
    *image_view;

  const char
    *artifact;

  double
    amount;

  DoublePixelPacket
    **pixels;

  MagickBooleanType
    status;

  ssize_t
    y;

  /*
    Distribute quantization error using Floyd-Steinberg.
  */
  pixels=AcquirePixelThreadSet(image->columns);
  if (pixels == (DoublePixelPacket **) NULL)
    return(MagickFalse);
  status=MagickTrue;
  amount=1.0;
  artifact=GetImageArtifact(image,"dither:diffusion-amount");
  if (artifact != (const char *) NULL)
    amount=StringToDoubleInterval(artifact,1.0);
  image_view=AcquireAuthenticCacheView(image,exception);
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    const int
      id = GetOpenMPThreadId();

    CubeInfo
      cube;

    DoublePixelPacket
      *current,
      *previous;

    register Quantum
      *magick_restrict q;

    register ssize_t
      x;

    size_t
      index;

    ssize_t
      v;

    if (status == MagickFalse)
      continue;
    q=GetCacheViewAuthenticPixels(image_view,0,y,image->columns,1,exception);
    if (q == (Quantum *) NULL)
      {
        status=MagickFalse;
        continue;
      }
    cube=(*cube_info);
    current=pixels[id]+(y & 0x01)*image->columns;
    previous=pixels[id]+((y+1) & 0x01)*image->columns;
    v=(ssize_t) ((y & 0x01) != 0 ? -1 : 1);
    for (x=0; x < (ssize_t) image->columns; x++)
    {
      DoublePixelPacket
        color,
        pixel;

      register ssize_t
        i;

      ssize_t
        u;

      u=(y & 0x01) != 0 ? (ssize_t) image->columns-1-x : x;
      AssociateAlphaPixel(image,&cube,q+u*GetPixelChannels(image),&pixel);
      if (x > 0)
        {
          pixel.red+=7.0*amount*current[u-v].red/16;
          pixel.green+=7.0*amount*current[u-v].green/16;
          pixel.blue+=7.0*amount*current[u-v].blue/16;
          if (cube.associate_alpha != MagickFalse)
            pixel.alpha+=7.0*amount*current[u-v].alpha/16;
        }
      if (y > 0)
        {
          if (x < (ssize_t) (image->columns-1))
            {
              pixel.red+=previous[u+v].red/16;
              pixel.green+=previous[u+v].green/16;
              pixel.blue+=previous[u+v].blue/16;
              if (cube.associate_alpha != MagickFalse)
                pixel.alpha+=previous[u+v].alpha/16;
            }
          pixel.red+=5.0*amount*previous[u].red/16;
          pixel.green+=5.0*amount*previous[u].green/16;
          pixel.blue+=5.0*amount*previous[u].blue/16;
          if (cube.associate_alpha != MagickFalse)
            pixel.alpha+=5.0*amount*previous[u].alpha/16;
          if (x > 0)
            {
              pixel.red+=3.0*amount*previous[u-v].red/16;
              pixel.green+=3.0*amount*previous[u-v].green/16;
              pixel.blue+=3.0*amount*previous[u-v].blue/16;
              if (cube.associate_alpha != MagickFalse)
                pixel.alpha+=3.0*amount*previous[u-v].alpha/16;
            }
        }
      pixel.red=(double) ClampPixel(pixel.red);
      pixel.green=(double) ClampPixel(pixel.green);
      pixel.blue=(double) ClampPixel(pixel.blue);
      if (cube.associate_alpha != MagickFalse)
        pixel.alpha=(double) ClampPixel(pixel.alpha);
      i=CacheOffset(&cube,&pixel);
      if (cube.cache[i] < 0)
        {
          register NodeInfo
            *node_info;

          register size_t
            node_id;

          /*
            Identify the deepest node containing the pixel's color.
          */
          node_info=cube.root;
          for (index=MaxTreeDepth-1; (ssize_t) index > 0; index--)
          {
            node_id=ColorToNodeId(&cube,&pixel,index);
            if (node_info->child[node_id] == (NodeInfo *) NULL)
              break;
            node_info=node_info->child[node_id];
          }
          /*
            Find closest color among siblings and their children.
          */
          cube.target=pixel;
          cube.distance=(double) (4.0*(QuantumRange+1.0)*(QuantumRange+1.0)+
            1.0);
          ClosestColor(image,&cube,node_info->parent);
          cube.cache[i]=(ssize_t) cube.color_number;
        }
      /*
        Assign pixel to closest colormap entry.
      */
      index=(size_t) cube.cache[i];
      if (image->storage_class == PseudoClass)
        SetPixelIndex(image,(Quantum) index,q+u*GetPixelChannels(image));
      if (cube.quantize_info->measure_error == MagickFalse)
        {
          SetPixelRed(image,ClampToQuantum(image->colormap[index].red),
            q+u*GetPixelChannels(image));
          SetPixelGreen(image,ClampToQuantum(image->colormap[index].green),
            q+u*GetPixelChannels(image));
          SetPixelBlue(image,ClampToQuantum(image->colormap[index].blue),
            q+u*GetPixelChannels(image));
          if (cube.associate_alpha != MagickFalse)
            SetPixelAlpha(image,ClampToQuantum(image->colormap[index].alpha),
              q+u*GetPixelChannels(image));
        }
      if (SyncCacheViewAuthenticPixels(image_view,exception) == MagickFalse)
        status=MagickFalse;
      /*
        Store the error.
      */
      AssociateAlphaPixelInfo(&cube,image->colormap+index,&color);
      current[u].red=pixel.red-color.red;
      current[u].green=pixel.green-color.green;
      current[u].blue=pixel.blue-color.blue;
      if (cube.associate_alpha != MagickFalse)
        current[u].alpha=pixel.alpha-color.alpha;
      if (image->progress_monitor != (MagickProgressMonitor) NULL)
        {
          MagickBooleanType
            proceed;

          proceed=SetImageProgress(image,DitherImageTag,(MagickOffsetType) y,
            image->rows);
          if (proceed == MagickFalse)
            status=MagickFalse;
        }
    }
  }
  image_view=DestroyCacheView(image_view);
  pixels=DestroyPixelThreadSet(pixels);
  return(MagickTrue);
}