Commit message: https://github.com/ImageMagick/ImageMagick/issues/1720
CVE: CVE-2020-27759
CWE: CWE-190
Vulnerability description: In IntensityCompare() of /MagickCore/quantize.c, a double value was being casted to int and returned, which in some cases caused a value outside the range of type `int` to be returned. The flaw could be triggered by a crafted input file under certain conditions when processed by ImageMagick. Red Hat Product Security marked this as Low severity because although it could potentially lead to an impact to application availability, no specific impact was shown in this case. This flaw affects ImageMagick versions prior to 7.0.8-68.

Project: ImageMagick
File: quantize.c

MagickExport MagickBooleanType GetImageQuantizeError(Image *image,
  ExceptionInfo *exception)
{
  CacheView
    *image_view;

  double
    alpha,
    area,
    beta,
    distance,
    maximum_error,
    mean_error,
    mean_error_per_pixel;

  ssize_t
    index,
    y;

  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  image->total_colors=GetNumberColors(image,(FILE *) NULL,exception);
  (void) memset(&image->error,0,sizeof(image->error));
  if (image->storage_class == DirectClass)
    return(MagickTrue);
  alpha=1.0;
  beta=1.0;
  area=3.0*image->columns*image->rows;
  maximum_error=0.0;
  mean_error_per_pixel=0.0;
  mean_error=0.0;
  image_view=AcquireVirtualCacheView(image,exception);
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    register const Quantum
      *magick_restrict p;

    register ssize_t
      x;

    p=GetCacheViewVirtualPixels(image_view,0,y,image->columns,1,exception);
    if (p == (const Quantum *) NULL)
      break;
    for (x=0; x < (ssize_t) image->columns; x++)
    {
      index=(ssize_t) GetPixelIndex(image,p);
      if (image->alpha_trait == BlendPixelTrait)
        {
          alpha=(double) (QuantumScale*GetPixelAlpha(image,p));
          beta=(double) (QuantumScale*image->colormap[index].alpha);
        }
      distance=fabs((double) (alpha*GetPixelRed(image,p)-beta*
        image->colormap[index].red));
      mean_error_per_pixel+=distance;
      mean_error+=distance*distance;
      if (distance > maximum_error)
        maximum_error=distance;
      distance=fabs((double) (alpha*GetPixelGreen(image,p)-beta*
        image->colormap[index].green));
      mean_error_per_pixel+=distance;
      mean_error+=distance*distance;
      if (distance > maximum_error)
        maximum_error=distance;
      distance=fabs((double) (alpha*GetPixelBlue(image,p)-beta*
        image->colormap[index].blue));
      mean_error_per_pixel+=distance;
      mean_error+=distance*distance;
      if (distance > maximum_error)
        maximum_error=distance;
      p+=GetPixelChannels(image);
    }
  }
  image_view=DestroyCacheView(image_view);
  image->error.mean_error_per_pixel=(double) mean_error_per_pixel/area;
  image->error.normalized_mean_error=(double) QuantumScale*QuantumScale*
    mean_error/area;
  image->error.normalized_maximum_error=(double) QuantumScale*maximum_error;
  return(MagickTrue);
}

Project: ImageMagick
File: quantize.c

MagickExport MagickBooleanType QuantizeImage(const QuantizeInfo *quantize_info,
  Image *image,ExceptionInfo *exception)
{
  CubeInfo
    *cube_info;

  MagickBooleanType
    status;

  size_t
    depth,
    maximum_colors;

  assert(quantize_info != (const QuantizeInfo *) NULL);
  assert(quantize_info->signature == MagickCoreSignature);
  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  maximum_colors=quantize_info->number_colors;
  if (maximum_colors == 0)
    maximum_colors=MaxColormapSize;
  if (maximum_colors > MaxColormapSize)
    maximum_colors=MaxColormapSize;
  if (image->alpha_trait != BlendPixelTrait)
    {
      if (SetImageGray(image,exception) != MagickFalse)
        (void) SetGrayscaleImage(image,exception);
    }
  if ((quantize_info->dither_method == NoDitherMethod) &&
      (image->storage_class == PseudoClass) &&
      (image->colors <= maximum_colors))
    {
      if ((quantize_info->colorspace != UndefinedColorspace) &&
          (quantize_info->colorspace != CMYKColorspace))
        (void) TransformImageColorspace(image,quantize_info->colorspace,
          exception);
      return(MagickTrue);
    }
  depth=quantize_info->tree_depth;
  if (depth == 0)
    {
      size_t
        colors;

      /*
        Depth of color tree is: Log4(colormap size)+2.
      */
      colors=maximum_colors;
      for (depth=1; colors != 0; depth++)
        colors>>=2;
      if ((quantize_info->dither_method != NoDitherMethod) && (depth > 2))
        depth--;
      if ((image->alpha_trait == BlendPixelTrait) && (depth > 5))
        depth--;
      if (SetImageGray(image,exception) != MagickFalse)
        depth=MaxTreeDepth;
    }
  /*
    Initialize color cube.
  */
  cube_info=GetCubeInfo(quantize_info,depth,maximum_colors);
  if (cube_info == (CubeInfo *) NULL)
    ThrowBinaryException(ResourceLimitError,"MemoryAllocationFailed",
      image->filename);
  status=ClassifyImageColors(cube_info,image,exception);
  if (status != MagickFalse)
    {
      /*
        Reduce the number of colors in the image if it contains more than the
        maximum, otherwise we can disable dithering to improve the performance.
      */
      if (cube_info->colors > cube_info->maximum_colors)
        ReduceImageColors(image,cube_info);
      else
        cube_info->quantize_info->dither_method=NoDitherMethod;
      status=AssignImageColors(image,cube_info,exception);
    }
  DestroyCubeInfo(cube_info);
  return(status);
}

Project: ImageMagick
File: quantize.c

static MagickBooleanType SetGrayscaleImage(Image *image,
  ExceptionInfo *exception)
{
  CacheView
    *image_view;

  MagickBooleanType
    status;

  PixelInfo
    *colormap;

  register ssize_t
    i;

  size_t
    extent;

  ssize_t
    *colormap_index,
    j,
    y;

  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->type != GrayscaleType)
    (void) TransformImageColorspace(image,GRAYColorspace,exception);
  extent=MagickMax(image->colors+1,MagickMax(MaxColormapSize,MaxMap+1));
  colormap_index=(ssize_t *) AcquireQuantumMemory(extent,
    sizeof(*colormap_index));
  if (colormap_index == (ssize_t *) NULL)
    ThrowBinaryException(ResourceLimitError,"MemoryAllocationFailed",
      image->filename);
  if (image->storage_class != PseudoClass)
    {
      (void) memset(colormap_index,(-1),extent*sizeof(*colormap_index));
      if (AcquireImageColormap(image,MaxColormapSize,exception) == MagickFalse)
        {
          colormap_index=(ssize_t *) RelinquishMagickMemory(colormap_index);
          ThrowBinaryException(ResourceLimitError,"MemoryAllocationFailed",
            image->filename);
        }
      image->colors=0;
      status=MagickTrue;
      image_view=AcquireAuthenticCacheView(image,exception);
#if defined(MAGICKCORE_OPENMP_SUPPORT)
      #pragma omp parallel for schedule(static) shared(status) \
        magick_number_threads(image,image,image->rows,1)
#endif
      for (y=0; y < (ssize_t) image->rows; y++)
      {
        register Quantum
          *magick_restrict q;

        register ssize_t
          x;

        if (status == MagickFalse)
          continue;
        q=GetCacheViewAuthenticPixels(image_view,0,y,image->columns,1,
          exception);
        if (q == (Quantum *) NULL)
          {
            status=MagickFalse;
            continue;
          }
        for (x=0; x < (ssize_t) image->columns; x++)
        {
          register size_t
            intensity;

          intensity=ScaleQuantumToMap(GetPixelRed(image,q));
          if (colormap_index[intensity] < 0)
            {
#if defined(MAGICKCORE_OPENMP_SUPPORT)
              #pragma omp critical (MagickCore_SetGrayscaleImage)
#endif
              if (colormap_index[intensity] < 0)
                {
                  colormap_index[intensity]=(ssize_t) image->colors;
                  image->colormap[image->colors].red=(double)
                    GetPixelRed(image,q);
                  image->colormap[image->colors].green=(double)
                    GetPixelGreen(image,q);
                  image->colormap[image->colors].blue=(double)
                    GetPixelBlue(image,q);
                  image->colors++;
               }
            }
          SetPixelIndex(image,(Quantum) colormap_index[intensity],q);
          q+=GetPixelChannels(image);
        }
        if (SyncCacheViewAuthenticPixels(image_view,exception) == MagickFalse)
          status=MagickFalse;
      }
      image_view=DestroyCacheView(image_view);
    }
  (void) memset(colormap_index,0,extent*sizeof(*colormap_index));
  for (i=0; i < (ssize_t) image->colors; i++)
    image->colormap[i].alpha=(double) i;
  qsort((void *) image->colormap,image->colors,sizeof(PixelInfo),
    IntensityCompare);
  colormap=(PixelInfo *) AcquireQuantumMemory(image->colors,sizeof(*colormap));
  if (colormap == (PixelInfo *) NULL)
    {
      colormap_index=(ssize_t *) RelinquishMagickMemory(colormap_index);
      ThrowBinaryException(ResourceLimitError,"MemoryAllocationFailed",
        image->filename);
    }
  j=0;
  colormap[j]=image->colormap[0];
  for (i=0; i < (ssize_t) image->colors; i++)
  {
    if (IsPixelInfoEquivalent(&colormap[j],&image->colormap[i]) == MagickFalse)
      {
        j++;
        colormap[j]=image->colormap[i];
      }
    colormap_index[(ssize_t) image->colormap[i].alpha]=j;
  }
  image->colors=(size_t) (j+1);
  image->colormap=(PixelInfo *) RelinquishMagickMemory(image->colormap);
  image->colormap=colormap;
  status=MagickTrue;
  image_view=AcquireAuthenticCacheView(image,exception);
#if defined(MAGICKCORE_OPENMP_SUPPORT)
  #pragma omp parallel for schedule(static) shared(status) \
    magick_number_threads(image,image,image->rows,1)
#endif
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    register Quantum
      *magick_restrict q;

    register ssize_t
      x;

    if (status == MagickFalse)
      continue;
    q=GetCacheViewAuthenticPixels(image_view,0,y,image->columns,1,exception);
    if (q == (Quantum *) NULL)
      {
        status=MagickFalse;
        continue;
      }
    for (x=0; x < (ssize_t) image->columns; x++)
    {
      SetPixelIndex(image,(Quantum) colormap_index[ScaleQuantumToMap(
        GetPixelIndex(image,q))],q);
      q+=GetPixelChannels(image);
    }
    if (SyncCacheViewAuthenticPixels(image_view,exception) == MagickFalse)
      status=MagickFalse;
  }
  image_view=DestroyCacheView(image_view);
  colormap_index=(ssize_t *) RelinquishMagickMemory(colormap_index);
  image->type=GrayscaleType;
  if (SetImageMonochrome(image,exception) != MagickFalse)
    image->type=BilevelType;
  return(status);
}