Vulnerability: In /MagickCore/statistic.c, there are several areas in ApplyEvaluateOperator() where a size_t cast should have been a ssize_t cast, which causes out-of-range values under some circumstances when a crafted input file is processed by ImageMagick. Red Hat Product Security marked this as Low severity because although it could potentially lead to an impact to application availability, no specific impact was shown in this case. This flaw affects ImageMagick versions prior to 6.9.10-69.
Vulnerability ID: CVE-2020-27764
CWE: CWE-190
Severity score: 2.9
Project: https://github.com/ImageMagick/ImageMagick6
Published: 2020-12-03
File: magick/statistic.c
Commit message: https://github.com/ImageMagick/ImageMagick/issues/1735
Commit author: Cristy

Vulnerable version (before fix):
 static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,     }     case AndEvaluateOperator:     {       result=(MagickRealType) ((size_t) pixel & (size_t) (value+0.5));       break;     }     case CosineEvaluateOperator: static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,     }     case LeftShiftEvaluateOperator:     {       result=(MagickRealType) ((size_t) pixel << (size_t) (value+0.5));       break;     }     case LogEvaluateOperator: static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,     }     case OrEvaluateOperator:     {       result=(MagickRealType) ((size_t) pixel | (size_t) (value+0.5));       break;     }     case PoissonNoiseEvaluateOperator: static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,     }     case RightShiftEvaluateOperator:     {       result=(MagickRealType) ((size_t) pixel >> (size_t) (value+0.5));       break;     }     case RootMeanSquareEvaluateOperator: static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,     }     case XorEvaluateOperator:     {       result=(MagickRealType) ((size_t) pixel ^ (size_t) (value+0.5));       break;     }   }

Fixed version (after fix):
 static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,     }     case AndEvaluateOperator:     {       result=(MagickRealType) ((ssize_t) pixel & (ssize_t) (value+0.5));       break;     }     case CosineEvaluateOperator: static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,     }     case LeftShiftEvaluateOperator:     {       result=(MagickRealType) ((ssize_t) pixel << (ssize_t) (value+0.5));       break;     }     case LogEvaluateOperator: static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,     }     case OrEvaluateOperator:     {       result=(MagickRealType) ((ssize_t) pixel | (ssize_t) (value+0.5));       break;     }     case PoissonNoiseEvaluateOperator: static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,     }     case RightShiftEvaluateOperator:     {       result=(MagickRealType) ((ssize_t) pixel >> (ssize_t) (value+0.5));       break;     }     case RootMeanSquareEvaluateOperator: static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,     }     case XorEvaluateOperator:     {       result=(MagickRealType) ((ssize_t) pixel ^ (ssize_t) (value+0.5));       break;     }   }

Patch:
patch: @@ -259,7 +259,7 @@ static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,
     }
     case AndEvaluateOperator:
     {
-      result=(MagickRealType) ((size_t) pixel & (size_t) (value+0.5));
+      result=(MagickRealType) ((ssize_t) pixel & (ssize_t) (value+0.5));
       break;
     }
     case CosineEvaluateOperator:
@@ -299,7 +299,7 @@ static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,
     }
     case LeftShiftEvaluateOperator:
     {
-      result=(MagickRealType) ((size_t) pixel << (size_t) (value+0.5));
+      result=(MagickRealType) ((ssize_t) pixel << (ssize_t) (value+0.5));
       break;
     }
     case LogEvaluateOperator:
@@ -342,7 +342,7 @@ static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,
     }
     case OrEvaluateOperator:
     {
-      result=(MagickRealType) ((size_t) pixel | (size_t) (value+0.5));
+      result=(MagickRealType) ((ssize_t) pixel | (ssize_t) (value+0.5));
       break;
     }
     case PoissonNoiseEvaluateOperator:
@@ -359,7 +359,7 @@ static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,
     }
     case RightShiftEvaluateOperator:
     {
-      result=(MagickRealType) ((size_t) pixel >> (size_t) (value+0.5));
+      result=(MagickRealType) ((ssize_t) pixel >> (ssize_t) (value+0.5));
       break;
     }
     case RootMeanSquareEvaluateOperator:
@@ -413,7 +413,7 @@ static MagickRealType ApplyEvaluateOperator(RandomInfo *random_info,
     }
     case XorEvaluateOperator:
     {
-      result=(MagickRealType) ((size_t) pixel ^ (size_t) (value+0.5));
+      result=(MagickRealType) ((ssize_t) pixel ^ (ssize_t) (value+0.5));
       break;
     }
   }
path: magick/statistic.c