Commit message: https://github.com/ImageMagick/ImageMagick/issues/1712
CVE: CVE-2020-27757
CWE: CWE-190
Vulnerability description: A floating point math calculation in ScaleAnyToQuantum() of /MagickCore/quantum-private.h could lead to undefined behavior in the form of a value outside the range of type unsigned long long. The flaw could be triggered by a crafted input file under certain conditions when it is processed by ImageMagick. Red Hat Product Security marked this as Low because although it could potentially lead to an impact to application availability, no specific impact was shown in this case. This flaw affects ImageMagick versions prior to 7.0.8-68.

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleLongToQuantum(const unsigned int value)
{
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) ((value)/MagickULLConstant(65537)));
#else
  return((Quantum) (value/65537.0));
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleShortToQuantum(const unsigned short value)
{
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) ((value+128U)/257U));
#else
  return((Quantum) (value/257.0));
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleCharToQuantum(const unsigned char value)
{
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) (16843009UL*value));
#else
  return((Quantum) (16843009.0*value));
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleLongToQuantum(const unsigned int value)
{
  return((Quantum) (4294967297.0*value));
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleMapToQuantum(const MagickRealType value)
{
  if (value <= 0.0)
    return((Quantum) 0);
  if (value >= MaxMap)
    return(QuantumRange);
  return((Quantum) (281479271743489.0*value));
}

Project: ImageMagick
File: quantum-private.h

static inline unsigned char *PopShortPixel(const EndianType endian,
  const unsigned short pixel,unsigned char *pixels)
{
  register unsigned int
    quantum;

  quantum=pixel;
  if (endian == LSBEndian)
    {
      *pixels++=(unsigned char) (quantum);
      *pixels++=(unsigned char) (quantum >> 8);
      return(pixels);
    }
  *pixels++=(unsigned char) (quantum >> 8);
  *pixels++=(unsigned char) (quantum);
  return(pixels);
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleCharToQuantum(const unsigned char value)
{
  return((Quantum) (72340172838076673.0*value));
}

Project: ImageMagick
File: quantum-private.h

static inline unsigned char *PopCharPixel(const unsigned char pixel,
  unsigned char *pixels)
{
  *pixels++=pixel;
  return(pixels);
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleLongLongToQuantum(const MagickSizeType value)
{
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) ((value)/MagickULLConstant(281479271743489)));
#else
  return((Quantum) (value/281479271743489.0));
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline float HalfToSinglePrecision(const unsigned short half)
{
#define ExponentBias  (127-15)
#define ExponentMask  0x7c00
#define ExponentShift  23
#define SignBitShift  31
#define SignificandShift  13
#define SignificandMask  0x00000400

  typedef union _SinglePrecision
  {
    unsigned int
      fixed_point;

    float
      single_precision;
  } SinglePrecision;

  register unsigned int
    exponent,
    significand,
    sign_bit;

  SinglePrecision
    map;

  unsigned int
    value;

  /*
    The IEEE 754 standard specifies half precision as having:

      Sign bit: 1 bit
      Exponent width: 5 bits
      Significand precision: 11 (10 explicitly stored)
  */
  sign_bit=(unsigned int) ((half >> 15) & 0x00000001);
  exponent=(unsigned int) ((half >> 10) & 0x0000001f);
  significand=(unsigned int) (half & 0x000003ff);
  if (exponent == 0)
    {
      if (significand == 0)
        value=sign_bit << SignBitShift;
      else
        {
          while ((significand & SignificandMask) == 0)
          {
            significand<<=1;
            exponent--;
          }
          exponent++;
          significand&=(~SignificandMask);
          exponent+=ExponentBias;
          value=(sign_bit << SignBitShift) | (exponent << ExponentShift) |
            (significand << SignificandShift);
        }
    }
  else
    if (exponent == SignBitShift)
      {
        value=(sign_bit << SignBitShift) | 0x7f800000;
        if (significand != 0)
          value|=(significand << SignificandShift);
      }
    else
      {
        exponent+=ExponentBias;
        significand<<=SignificandShift;
        value=(sign_bit << SignBitShift) | (exponent << ExponentShift) |
          significand;
      }
  map.fixed_point=value;
  return(map.single_precision);
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleAnyToQuantum(const QuantumAny quantum,
  const QuantumAny range)
{
  if (quantum > range)
    return(QuantumRange);
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) (((double) QuantumRange*quantum)*
    PerceptibleReciprocal((double) range)+0.5));
#else
  return((Quantum) (((double) QuantumRange*quantum)*
    PerceptibleReciprocal((double) range)));
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleCharToQuantum(const unsigned char value)
{
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) (257U*value));
#else
  return((Quantum) (257.0*value));
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline const unsigned char *PushFloatPixel(const EndianType endian,
  const unsigned char *pixels,MagickFloatType *pixel)
{
  union
  {
    unsigned int
      unsigned_value;

    MagickFloatType
      float_value;
  } quantum;

  if (endian == LSBEndian)
    {
      quantum.unsigned_value=((unsigned int) *pixels++);
      quantum.unsigned_value|=((unsigned int) *pixels++ << 8);
      quantum.unsigned_value|=((unsigned int) *pixels++ << 16);
      quantum.unsigned_value|=((unsigned int) *pixels++ << 24);
      *pixel=quantum.float_value;
      return(pixels);
    }
  quantum.unsigned_value=((unsigned int) *pixels++ << 24);
  quantum.unsigned_value|=((unsigned int) *pixels++ << 16);
  quantum.unsigned_value|=((unsigned int) *pixels++ << 8);
  quantum.unsigned_value|=((unsigned int) *pixels++);
  *pixel=quantum.float_value;
  return(pixels);
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleMapToQuantum(const MagickRealType value)
{
  if (value <= 0.0)
    return((Quantum) 0);
  if (value >= (Quantum) MaxMap)
    return(QuantumRange);
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) (65537.0*value+0.5));
#else
  return((Quantum) (65537.0*value));
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline const unsigned char *PushShortPixel(const EndianType endian,
  const unsigned char *pixels,unsigned short *pixel)
{
  register unsigned int
    quantum;

  if (endian == LSBEndian)
    {
      quantum=(unsigned int) *pixels++;
      quantum|=(unsigned int) (*pixels++ << 8);
      *pixel=(unsigned short) (quantum & 0xffff);
      return(pixels);
    }
  quantum=(unsigned int) (*pixels++ << 8);
  quantum|=(unsigned int) *pixels++;
  *pixel=(unsigned short) (quantum & 0xffff);
  return(pixels);
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleLongLongToQuantum(const MagickSizeType value)
{
  return((Quantum) (value));
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleLongLongToQuantum(const MagickSizeType value)
{
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) (value/MagickULLConstant(72340172838076673)));
#else
  return((Quantum) (value/72340172838076673.0));
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline unsigned char *PopLongPixel(const EndianType endian,
  const unsigned int pixel,unsigned char *pixels)
{
  register unsigned int
    quantum;

  quantum=(unsigned int) pixel;
  if (endian == LSBEndian)
    {
      *pixels++=(unsigned char) (quantum);
      *pixels++=(unsigned char) (quantum >> 8);
      *pixels++=(unsigned char) (quantum >> 16);
      *pixels++=(unsigned char) (quantum >> 24);
      return(pixels);
    }
  *pixels++=(unsigned char) (quantum >> 24);
  *pixels++=(unsigned char) (quantum >> 16);
  *pixels++=(unsigned char) (quantum >> 8);
  *pixels++=(unsigned char) (quantum);
  return(pixels);
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleLongToQuantum(const unsigned int value)
{
  return((Quantum) value);
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleShortToQuantum(const unsigned short value)
{
  return((Quantum) (281479271743489.0*value));
}

Project: ImageMagick
File: quantum-private.h

static inline unsigned short SinglePrecisionToHalf(const float value)
{
  typedef union _SinglePrecision
  {
    unsigned int
      fixed_point;

    float
      single_precision;
  } SinglePrecision;

  register int
    exponent;

  register unsigned int
    significand,
    sign_bit;

  SinglePrecision
    map;

  unsigned short
    half;

  /*
    The IEEE 754 standard specifies half precision as having:

      Sign bit: 1 bit
      Exponent width: 5 bits
      Significand precision: 11 (10 explicitly stored)
  */
  map.single_precision=value;
  sign_bit=(map.fixed_point >> 16) & 0x00008000;
  exponent=(int) ((map.fixed_point >> ExponentShift) & 0x000000ff)-ExponentBias;
  significand=map.fixed_point & 0x007fffff;
  if (exponent <= 0)
    {
      int
        shift;

      if (exponent < -10)
        return((unsigned short) sign_bit);
      significand=significand | 0x00800000;
      shift=(int) (14-exponent);
      significand=(unsigned int) ((significand+((1 << (shift-1))-1)+
        ((significand >> shift) & 0x01)) >> shift);
      return((unsigned short) (sign_bit | significand));
    }
  else
    if (exponent == (0xff-ExponentBias))
      {
        if (significand == 0)
          return((unsigned short) (sign_bit | ExponentMask));
        else
          {
            significand>>=SignificandShift;
            half=(unsigned short) (sign_bit | significand |
              (significand == 0) | ExponentMask);
            return(half);
          }
      }
  significand=significand+((significand >> SignificandShift) & 0x01)+0x00000fff;
  if ((significand & 0x00800000) != 0)
    {
      significand=0;
      exponent++;
    }
  if (exponent > 30)
    {
      float
        alpha;

      register int
        i;

      /*
        Float overflow.
      */
      alpha=1.0e10;
      for (i=0; i < 10; i++)
        alpha*=alpha;
      return((unsigned short) (sign_bit | ExponentMask));
    }
  half=(unsigned short) (sign_bit | (exponent << 10) |
    (significand >> SignificandShift));
  return(half);
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleLongToQuantum(const unsigned int value)
{
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) ((value)/16843009UL));
#else
  return((Quantum) (value/16843009.0));
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleMapToQuantum(const MagickRealType value)
{
  if (value <= 0.0)
    return((Quantum) 0);
  if (value >= MaxMap)
    return(QuantumRange);
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) (value+0.5));
#else
  return((Quantum) value);
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline const unsigned char *PushCharPixel(const unsigned char *pixels,
  unsigned char *pixel)
{
  *pixel=(*pixels++);
  return(pixels);
}

Project: ImageMagick
File: quantum-private.h

static inline MagickSizeType GetQuantumRange(const size_t depth)
{
  MagickSizeType
    one;

  size_t
    max_depth;

  if (depth == 0)
    return(0);
  one=1;
  max_depth=8*sizeof(MagickSizeType);
  return((MagickSizeType) ((one << (MagickMin(depth,max_depth)-1))+
    ((one << (MagickMin(depth,max_depth)-1))-1)));
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleShortToQuantum(const unsigned short value)
{
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) (65537UL*value));
#else
  return((Quantum) (65537.0*value));
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleCharToQuantum(const unsigned char value)
{
  return((Quantum) value);
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleLongLongToQuantum(const MagickSizeType value)
{
#if !defined(MAGICKCORE_HDRI_SUPPORT)
  return((Quantum) ((value)/MagickULLConstant(4294967297)));
#else
  return((Quantum) (value/4294967297.0));
#endif
}

Project: ImageMagick
File: quantum-private.h

static inline const unsigned char *PushLongPixel(const EndianType endian,
  const unsigned char *pixels,unsigned int *pixel)
{
  register unsigned int
    quantum;

  if (endian == LSBEndian)
    {
      quantum=((unsigned int) *pixels++);
      quantum|=((unsigned int) *pixels++ << 8);
      quantum|=((unsigned int) *pixels++ << 16);
      quantum|=((unsigned int) *pixels++ << 24);
      *pixel=quantum;
      return(pixels);
    }
  quantum=((unsigned int) *pixels++ << 24);
  quantum|=((unsigned int) *pixels++ << 16);
  quantum|=((unsigned int) *pixels++ << 8);
  quantum|=((unsigned int) *pixels++);
  *pixel=quantum;
  return(pixels);
}

Project: ImageMagick
File: quantum-private.h

static inline Quantum ScaleShortToQuantum(const unsigned short value)
{
  return((Quantum) value);
}