CVE-2016-9535 (published 2016-11-22T19:59Z)

tif_predict.h and tif_predict.c in libtiff 4.0.6 have assertions that can lead to assertion failures in debug mode, or buffer overflows in release mode, when dealing with unusual tile size like YCbCr with subsampling. Reported as MSVR 35105, aka "Predictor heap-buffer-overflow."

Severity: HIGH
CVSS v2 base score: 7.5 (AV:N/AC:L/Au:N/C:P/I:P/A:P)
CVSS v3 base score: 9.8, CRITICAL (CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H)

Weakness information:
cwe_id: CWE-119
cwe_name: Improper Restriction of Operations within the Bounds of a Memory Buffer
description: The product performs operations on a memory buffer, but it reads from or writes to a memory location outside the buffer's intended boundary. This may result in read or write operations on unexpected memory locations that could be linked to other variables, data structures, or internal program data.
extended_description: 
            Certain languages allow direct addressing of memory locations and do not automatically ensure that these locations are valid for the memory buffer that is being referenced. This can cause read or write operations to be performed on memory locations that may be associated with other variables, data structures, or internal program data.
         
         

Fix information:
repo_url: https://github.com/vadz/libtiff
commit_details: repo_url: https://github.com/vadz/libtiff
author: erouault
author_date: 2016-10-31 17:24:26+00:00
msg: * libtiff/tif_predict.h, libtiff/tif_predict.c:
Replace assertions by runtime checks to avoid assertions in debug mode,
or buffer overflows in release mode. Can happen when dealing with
unusual tile size like YCbCr with subsampling. Reported as MSVR 35105
by Axel Souchet	& Vishal Chauhan from the MSRC Vulnerabilities & Mitigations
team.
repository: repo_url: https://github.com/vadz/libtiff
repo_name: vadz/libtiff
description: OBSOLETE and not updated any more libtiff mirror, see website for the new one
repo_language: C
owner: vadz
file_changes: filename: ChangeLog
change_type: ModificationType.MODIFY
diff: @@ -1,3 +1,12 @@
+2016-10-31 Even Rouault <even.rouault at spatialys.com>
+
+	* libtiff/tif_predict.h, libtiff/tif_predict.c:
+	Replace assertions by runtime checks to avoid assertions in debug mode,
+	or buffer overflows in release mode. Can happen when dealing with
+	unusual tile size like YCbCr with subsampling. Reported as MSVR 35105
+	by Axel Souchet	& Vishal Chauhan from the MSRC Vulnerabilities & Mitigations
+	team.
+
 2016-10-26 Even Rouault <even.rouault at spatialys.com>
 
 	* tools/fax2tiff.c: fix segfault when specifying -r without

programming_language: Markdown
filename: tif_predict.c
change_type: ModificationType.MODIFY
diff: @@ -34,18 +34,18 @@
 
 #define	PredictorState(tif)	((TIFFPredictorState*) (tif)->tif_data)
 
-static void horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc);
-static void horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
-static void horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
-static void swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
-static void swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
-static void horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc);
-static void horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
-static void horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
-static void swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
-static void swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
-static void fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc);
-static void fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc);
+static int fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc);
 static int PredictorDecodeRow(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s);
 static int PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s);
 static int PredictorEncodeRow(TIFF* tif, uint8* bp, tmsize_t cc, uint16 s);
@@ -273,13 +273,19 @@ PredictorSetupEncode(TIFF* tif)
 /* - when storing into the byte stream, we explicitly mask with 0xff so */
 /*   as to make icc -check=conversions happy (not necessary by the standard) */
 
-static void
+static int
 horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
 	tmsize_t stride = PredictorState(tif)->stride;
 
 	unsigned char* cp = (unsigned char*) cp0;
-	assert((cc%stride)==0);
+    if((cc%stride)!=0)
+    {
+        TIFFErrorExt(tif->tif_clientdata, "horAcc8",
+                     "%s", "(cc%stride)!=0");
+        return 0;
+    }
+
 	if (cc > stride) {
 		/*
 		 * Pipeline the most common cases.
@@ -321,26 +327,32 @@ horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc)
 			} while (cc>0);
 		}
 	}
+	return 1;
 }
 
-static void
+static int
 swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
 	uint16* wp = (uint16*) cp0;
 	tmsize_t wc = cc / 2;
 
         TIFFSwabArrayOfShort(wp, wc);
-        horAcc16(tif, cp0, cc);
+        return horAcc16(tif, cp0, cc);
 }
 
-static void
+static int
 horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
 	tmsize_t stride = PredictorState(tif)->stride;
 	uint16* wp = (uint16*) cp0;
 	tmsize_t wc = cc / 2;
 
-	assert((cc%(2*stride))==0);
+    if((cc%(2*stride))!=0)
+    {
+        TIFFErrorExt(tif->tif_clientdata, "horAcc16",
+                     "%s", "cc%(2*stride))!=0");
+        return 0;
+    }
 
 	if (wc > stride) {
 		wc -= stride;
@@ -349,26 +361,32 @@ horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)
 			wc -= stride;
 		} while (wc > 0);
 	}
+	return 1;
 }
 
-static void
+static int
 swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
 	uint32* wp = (uint32*) cp0;
 	tmsize_t wc = cc / 4;
 
         TIFFSwabArrayOfLong(wp, wc);
-	horAcc32(tif, cp0, cc);
+	return horAcc32(tif, cp0, cc);
 }
 
-static void
+static int
 horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
 	tmsize_t stride = PredictorState(tif)->stride;
 	uint32* wp = (uint32*) cp0;
 	tmsize_t wc = cc / 4;
 
-	assert((cc%(4*stride))==0);
+    if((cc%(4*stride))!=0)
+    {
+        TIFFErrorExt(tif->tif_clientdata, "horAcc32",
+                     "%s", "cc%(4*stride))!=0");
+        return 0;
+    }
 
 	if (wc > stride) {
 		wc -= stride;
@@ -377,12 +395,13 @@ horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)
 			wc -= stride;
 		} while (wc > 0);
 	}
+	return 1;
 }
 
 /*
  * Floating point predictor accumulation routine.
  */
-static void
+static int
 fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
 	tmsize_t stride = PredictorState(tif)->stride;
@@ -392,10 +411,15 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
 	uint8 *cp = (uint8 *) cp0;
 	uint8 *tmp = (uint8 *)_TIFFmalloc(cc);
 
-	assert((cc%(bps*stride))==0);
+    if(cc%(bps*stride)!=0)
+    {
+        TIFFErrorExt(tif->tif_clientdata, "fpAcc",
+                     "%s", "cc%(bps*stride))!=0");
+        return 0;
+    }
 
 	if (!tmp)
-		return;
+		return 0;
 
 	while (count > stride) {
 		REPEAT4(stride, cp[stride] =
@@ -417,6 +441,7 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
 		}
 	}
 	_TIFFfree(tmp);
+    return 1;
 }
 
 /*
@@ -432,8 +457,7 @@ PredictorDecodeRow(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)
 	assert(sp->decodepfunc != NULL);  
 
 	if ((*sp->decoderow)(tif, op0, occ0, s)) {
-		(*sp->decodepfunc)(tif, op0, occ0);
-		return 1;
+		return (*sp->decodepfunc)(tif, op0, occ0);
 	} else
 		return 0;
 }
@@ -456,10 +480,16 @@ PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)
 	if ((*sp->decodetile)(tif, op0, occ0, s)) {
 		tmsize_t rowsize = sp->rowsize;
 		assert(rowsize > 0);
-		assert((occ0%rowsize)==0);
+		if((occ0%rowsize) !=0)
+        {
+            TIFFErrorExt(tif->tif_clientdata, "PredictorDecodeTile",
+                         "%s", "occ0%rowsize != 0");
+            return 0;
+        }
 		assert(sp->decodepfunc != NULL);
 		while (occ0 > 0) {
-			(*sp->decodepfunc)(tif, op0, rowsize);
+			if( !(*sp->decodepfunc)(tif, op0, rowsize) )
+                return 0;
 			occ0 -= rowsize;
 			op0 += rowsize;
 		}
@@ -468,14 +498,19 @@ PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)
 		return 0;
 }
 
-static void
+static int
 horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
 	TIFFPredictorState* sp = PredictorState(tif);
 	tmsize_t stride = sp->stride;
 	unsigned char* cp = (unsigned char*) cp0;
 
-	assert((cc%stride)==0);
+    if((cc%stride)!=0)
+    {
+        TIFFErrorExt(tif->tif_clientdata, "horDiff8",
+                     "%s", "(cc%stride)!=0");
+        return 0;
+    }
 
 	if (cc > stride) {
 		cc -= stride;
@@ -513,9 +548,10 @@ horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc)
 			} while ((cc -= stride) > 0);
 		}
 	}
+	return 1;
 }
 
-static void
+static int
 horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
 	TIFFPredictorState* sp = PredictorState(tif);
@@ -523,7 +559,12 @@ horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
 	uint16 *wp = (uint16*) cp0;
 	tmsize_t wc = cc/2;
 
-	assert((cc%(2*stride))==0);
+    if((cc%(2*stride))!=0)
+    {
+        TIFFErrorExt(tif->tif_clientdata, "horDiff8",
+                     "%s", "(cc%(2*stride))!=0");
+        return 0;
+    }
 
 	if (wc > stride) {
 		wc -= stride;
@@ -533,20 +574,23 @@ horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
 			wc -= stride;
 		} while (wc > 0);
 	}
+	return 1;
 }
 
-static void
+static int
 swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
     uint16* wp = (uint16*) cp0;
     tmsize_t wc = cc / 2;
 
-    horDiff16(tif, cp0, cc);
+    if( !horDiff16(tif, cp0, cc) )
+        return 0;
 
     TIFFSwabArrayOfShort(wp, wc);
+    return 1;
 }
 
-static void
+static int
 horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
 	TIFFPredictorState* sp = PredictorState(tif);
@@ -554,7 +598,12 @@ horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
 	uint32 *wp = (uint32*) cp0;
 	tmsize_t wc = cc/4;
 
-	assert((cc%(4*stride))==0);
+    if((cc%(4*stride))!=0)
+    {
+        TIFFErrorExt(tif->tif_clientdata, "horDiff32",
+                     "%s", "(cc%(4*stride))!=0");
+        return 0;
+    }
 
 	if (wc > stride) {
 		wc -= stride;
@@ -564,23 +613,26 @@ horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
 			wc -= stride;
 		} while (wc > 0);
 	}
+	return 1;
 }
 
-static void
+static int
 swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
     uint32* wp = (uint32*) cp0;
     tmsize_t wc = cc / 4;
 
-    horDiff32(tif, cp0, cc);
+    if( !horDiff32(tif, cp0, cc) )
+        return 0;
 
     TIFFSwabArrayOfLong(wp, wc);
+    return 1;
 }
 
 /*
  * Floating point predictor differencing routine.
  */
-static void
+static int
 fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
 {
 	tmsize_t stride = PredictorState(tif)->stride;
@@ -590,10 +642,14 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
 	uint8 *cp = (uint8 *) cp0;
 	uint8 *tmp = (uint8 *)_TIFFmalloc(cc);
 
-	assert((cc%(bps*stride))==0);
-
+    if((cc%(bps*stride))!=0)
+    {
+        TIFFErrorExt(tif->tif_clientdata, "fpDiff",
+                     "%s", "(cc%(bps*stride))!=0");
+        return 0;
+    }
 	if (!tmp)
-		return;
+		return 0;
 
 	_TIFFmemcpy(tmp, cp0, cc);
 	for (count = 0; count < wc; count++) {
@@ -613,6 +669,7 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
 	cp += cc - stride - 1;
 	for (count = cc; count > stride; count -= stride)
 		REPEAT4(stride, cp[stride] = (unsigned char)((cp[stride] - cp[0])&0xff); cp--)
+    return 1;
 }
 
 static int
@@ -625,7 +682,8 @@ PredictorEncodeRow(TIFF* tif, uint8* bp, tmsize_t cc, uint16 s)
 	assert(sp->encoderow != NULL);
 
 	/* XXX horizontal differencing alters user's data XXX */
-	(*sp->encodepfunc)(tif, bp, cc);
+	if( !(*sp->encodepfunc)(tif, bp, cc) )
+        return 0;
 	return (*sp->encoderow)(tif, bp, cc, s);
 }
 
@@ -660,7 +718,12 @@ PredictorEncodeTile(TIFF* tif, uint8* bp0, tmsize_t cc0, uint16 s)
 
 	rowsize = sp->rowsize;
 	assert(rowsize > 0);
-	assert((cc0%rowsize)==0);
+	if((cc0%rowsize)!=0)
+    {
+        TIFFErrorExt(tif->tif_clientdata, "PredictorEncodeTile",
+                     "%s", "(cc0%rowsize)!=0");
+        return 0;
+    }
 	while (cc > 0) {
 		(*sp->encodepfunc)(tif, bp, rowsize);
 		cc -= rowsize;

programming_language: C
filename: tif_predict.h
change_type: ModificationType.MODIFY
diff: @@ -30,6 +30,8 @@
  * ``Library-private'' Support for the Predictor Tag
  */
 
+typedef int (*TIFFEncodeDecodeMethod)(TIFF* tif, uint8* buf, tmsize_t size);
+
 /*
  * Codecs that want to support the Predictor tag must place
  * this structure first in their private state block so that
@@ -43,12 +45,12 @@ typedef struct {
 	TIFFCodeMethod  encoderow;	/* parent codec encode/decode row */
 	TIFFCodeMethod  encodestrip;	/* parent codec encode/decode strip */
 	TIFFCodeMethod  encodetile;	/* parent codec encode/decode tile */ 
-	TIFFPostMethod  encodepfunc;	/* horizontal differencer */
+	TIFFEncodeDecodeMethod  encodepfunc;	/* horizontal differencer */
 
 	TIFFCodeMethod  decoderow;	/* parent codec encode/decode row */
 	TIFFCodeMethod  decodestrip;	/* parent codec encode/decode strip */
 	TIFFCodeMethod  decodetile;	/* parent codec encode/decode tile */ 
-	TIFFPostMethod  decodepfunc;	/* horizontal accumulator */
+	TIFFEncodeDecodeMethod  decodepfunc;	/* horizontal accumulator */
 
 	TIFFVGetMethod  vgetparent;	/* super-class method */
 	TIFFVSetMethod  vsetparent;	/* super-class method */

programming_language: C
repo_url: https://github.com/vadz/libtiff
commit_details: repo_url: https://github.com/vadz/libtiff
author: erouault
author_date: 2016-11-04 09:19:13+00:00
msg: * libtiff/tif_predic.c: fix memory leaks in error code paths added in
previous commit (fix for MSVR 35105)
repository: repo_url: https://github.com/vadz/libtiff
repo_name: vadz/libtiff
description: OBSOLETE and not updated any more libtiff mirror, see website for the new one
repo_language: C
owner: vadz
file_changes: filename: ChangeLog
change_type: ModificationType.MODIFY
diff: @@ -1,3 +1,8 @@
+2016-11-04 Even Rouault <even.rouault at spatialys.com>
+
+	* libtiff/tif_predic.c: fix memory leaks in error code paths added in
+	previous commit (fix for MSVR 35105)
+
 2016-10-31 Even Rouault <even.rouault at spatialys.com>
 
 	* libtiff/tif_predict.h, libtiff/tif_predict.c:

programming_language: Markdown
filename: tif_predict.c
change_type: ModificationType.MODIFY
diff: @@ -409,7 +409,7 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
 	tmsize_t wc = cc / bps;
 	tmsize_t count = cc;
 	uint8 *cp = (uint8 *) cp0;
-	uint8 *tmp = (uint8 *)_TIFFmalloc(cc);
+	uint8 *tmp;
 
     if(cc%(bps*stride)!=0)
     {
@@ -418,6 +418,7 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
         return 0;
     }
 
+    tmp = (uint8 *)_TIFFmalloc(cc);
 	if (!tmp)
 		return 0;
 
@@ -640,7 +641,7 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
 	tmsize_t wc = cc / bps;
 	tmsize_t count;
 	uint8 *cp = (uint8 *) cp0;
-	uint8 *tmp = (uint8 *)_TIFFmalloc(cc);
+	uint8 *tmp;
 
     if((cc%(bps*stride))!=0)
     {
@@ -648,6 +649,8 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
                      "%s", "(cc%(bps*stride))!=0");
         return 0;
     }
+
+    tmp = (uint8 *)_TIFFmalloc(cc);
 	if (!tmp)
 		return 0;
 
@@ -722,6 +725,7 @@ PredictorEncodeTile(TIFF* tif, uint8* bp0, tmsize_t cc0, uint16 s)
     {
         TIFFErrorExt(tif->tif_clientdata, "PredictorEncodeTile",
                      "%s", "(cc0%rowsize)!=0");
+        _TIFFfree( working_copy );
         return 0;
     }
 	while (cc > 0) {

programming_language: C