CVE-2020-27764: Integer Overflow or Wraparound (CWE-190)
value: In /MagickCore/statistic.c, there are several areas in ApplyEvaluateOperator() where a size_t cast should have been a ssize_t cast, which causes out-of-range values under some circumstances when a crafted input file is processed by ImageMagick. Red Hat Product Security marked this as Low severity because although it could potentially lead to an impact to application availability, no specific impact was shown in this case. This flaw affects ImageMagick versions prior to 6.9.10-69.
CWE description: The product performs a calculation that can
         produce an integer overflow or wraparound when the logic
         assumes that the resulting value will always be larger than
         the original value. This occurs when an integer value is
         incremented to a value that is too large to store in the
         associated representation. When this occurs, the value may
         become a very small or negative number.
CVSS v3 base score: 3.3
Published: 2020-12-03T17:15Z
Repository: https://github.com/ImageMagick/ImageMagick6
Language: C
Files changed: magick/statistic.c
Commit date: 2019-10-09 18:44:16 -0400
Commit message: https://github.com/ImageMagick/ImageMagick/issues/1735

Diff: diff --git a/magick/statistic.c b/magick/statistic.c
index 7bc816360aa..87a9a42f46b 100644
--- a/magick/statistic.c
+++ b/magick/statistic.c
@@ -257,11 +257,11 @@ static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,
       result-=(QuantumRange+1.0)*floor((double) result/(QuantumRange+1.0));
       break;
     }
     case AndEvaluateOperator:
     {
-      result=(MagickRealType) ((size_t) pixel & (size_t) (value+0.5));
+      result=(MagickRealType) ((ssize_t) pixel & (ssize_t) (value+0.5));
       break;
     }
     case CosineEvaluateOperator:
     {
       result=(MagickRealType) (QuantumRange*(0.5*cos((double) (2.0*MagickPI*
@@ -297,11 +297,11 @@ static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,
         LaplacianNoise,value);
       break;
     }
     case LeftShiftEvaluateOperator:
     {
-      result=(MagickRealType) ((size_t) pixel << (size_t) (value+0.5));
+      result=(MagickRealType) ((ssize_t) pixel << (ssize_t) (value+0.5));
       break;
     }
     case LogEvaluateOperator:
     {
       if ((QuantumScale*pixel) >= MagickEpsilon)
@@ -340,11 +340,11 @@ static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,
       result=(MagickRealType) (value*pixel);
       break;
     }
     case OrEvaluateOperator:
     {
-      result=(MagickRealType) ((size_t) pixel | (size_t) (value+0.5));
+      result=(MagickRealType) ((ssize_t) pixel | (ssize_t) (value+0.5));
       break;
     }
     case PoissonNoiseEvaluateOperator:
     {
       result=(MagickRealType) GenerateDifferentialNoise(random_info,pixel,
@@ -357,11 +357,11 @@ static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,
         (double) value));
       break;
     }
     case RightShiftEvaluateOperator:
     {
-      result=(MagickRealType) ((size_t) pixel >> (size_t) (value+0.5));
+      result=(MagickRealType) ((ssize_t) pixel >> (ssize_t) (value+0.5));
       break;
     }
     case RootMeanSquareEvaluateOperator:
     {
       result=(MagickRealType) (pixel*pixel+value);
@@ -411,11 +411,11 @@ static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,
         UniformNoise,value);
       break;
     }
     case XorEvaluateOperator:
     {
-      result=(MagickRealType) ((size_t) pixel ^ (size_t) (value+0.5));
+      result=(MagickRealType) ((ssize_t) pixel ^ (ssize_t) (value+0.5));
       break;
     }
   }
   return(result);
 }

Vulnerable code:       result=(MagickRealType) ((size_t) pixel & (size_t) (value+0.5));
      result=(MagickRealType) ((size_t) pixel << (size_t) (value+0.5));
      result=(MagickRealType) ((size_t) pixel | (size_t) (value+0.5));
      result=(MagickRealType) ((size_t) pixel >> (size_t) (value+0.5));
      result=(MagickRealType) ((size_t) pixel ^ (size_t) (value+0.5));
Fixed code:       result=(MagickRealType) ((ssize_t) pixel & (ssize_t) (value+0.5));
      result=(MagickRealType) ((ssize_t) pixel << (ssize_t) (value+0.5));
      result=(MagickRealType) ((ssize_t) pixel | (ssize_t) (value+0.5));
      result=(MagickRealType) ((ssize_t) pixel >> (ssize_t) (value+0.5));
      result=(MagickRealType) ((ssize_t) pixel ^ (ssize_t) (value+0.5));