Commit message: gd2: handle corrupt images better (CVE-2016-3074)

Make sure we do some range checking on corrupted chunks.

Thanks to Hans Jerry Illikainen <hji@dyntopia.com> for indepth report
and reproducer information.  Made for easy test case writing :).
CVE: CVE-2016-3074
CWE: CWE-189
Vulnerability description: Integer signedness error in GD Graphics Library 2.1.1 (aka libgd or libgd2) allows remote attackers to cause a denial of service (crash) or potentially execute arbitrary code via crafted compressed gd2 data, which triggers a heap-based buffer overflow.

Project: libgd

BGD_DECLARE(gdImagePtr) gdImageCreateFromGd2 (FILE * inFile)
{
	gdIOCtx *in = gdNewFileCtx (inFile);
	gdImagePtr im;

	if (in == NULL) return NULL;
	im = gdImageCreateFromGd2Ctx (in);

	in->gd_free (in);

	return im;
}

Project: libgd

BGD_DECLARE(gdImagePtr) gdImageCreateFromGd2Ctx (gdIOCtxPtr in)
{
	int sx, sy;
	int i;
	int ncx, ncy, nc, cs, cx, cy;
	int x, y, ylo, yhi, xlo, xhi;
	int vers, fmt;
	t_chunk_info *chunkIdx = NULL;	/* So we can gdFree it with impunity. */
	unsigned char *chunkBuf = NULL;	/* So we can gdFree it with impunity. */
	int chunkNum = 0;
	int chunkMax = 0;
	uLongf chunkLen;
	int chunkPos = 0;
	int compMax = 0;
	int bytesPerPixel;
	char *compBuf = NULL;		/* So we can gdFree it with impunity. */

	gdImagePtr im;

	/* Get the header */
	im =
	    _gd2CreateFromFile (in, &sx, &sy, &cs, &vers, &fmt, &ncx, &ncy,
	                        &chunkIdx);

	if (im == NULL) {
		gdFree (chunkIdx);
		return 0;
	}

	bytesPerPixel = im->trueColor ? 4 : 1;
	nc = ncx * ncy;

	if (gd2_compressed (fmt)) {
		/* Find the maximum compressed chunk size. */
		compMax = 0;
		for (i = 0; (i < nc); i++) {
			if (chunkIdx[i].size > compMax) {
				compMax = chunkIdx[i].size;
			};
		};
		compMax++;

		/* Allocate buffers */
		chunkMax = cs * bytesPerPixel * cs;
		chunkBuf = gdCalloc (chunkMax, 1);
		if (!chunkBuf) {
			goto fail;
		}
		compBuf = gdCalloc (compMax, 1);
		if (!compBuf) {
			goto fail;
		}

		GD2_DBG (printf ("Largest compressed chunk is %d bytes\n", compMax));
	};

	/*      if ( (ncx != sx / cs) || (ncy != sy / cs)) { */
	/*              goto fail2; */
	/*      }; */

	/* Read the data... */
	for (cy = 0; (cy < ncy); cy++) {
		for (cx = 0; (cx < ncx); cx++) {

			ylo = cy * cs;
			yhi = ylo + cs;
			if (yhi > im->sy) {
				yhi = im->sy;
			};

			GD2_DBG (printf
			         ("Processing Chunk %d (%d, %d), y from %d to %d\n",
			          chunkNum, cx, cy, ylo, yhi));

			if (gd2_compressed (fmt)) {

				chunkLen = chunkMax;

				if (!_gd2ReadChunk (chunkIdx[chunkNum].offset,
				                    compBuf,
				                    chunkIdx[chunkNum].size,
				                    (char *) chunkBuf, &chunkLen, in)) {
					GD2_DBG (printf ("Error reading comproessed chunk\n"));
					goto fail;
				};

				chunkPos = 0;
			};

			for (y = ylo; (y < yhi); y++) {

				xlo = cx * cs;
				xhi = xlo + cs;
				if (xhi > im->sx) {
					xhi = im->sx;
				};
				/*GD2_DBG(printf("y=%d: ",y)); */
				if (!gd2_compressed (fmt)) {
					for (x = xlo; x < xhi; x++) {

						if (im->trueColor) {
							if (!gdGetInt (&im->tpixels[y][x], in)) {
								/*printf("EOF while reading\n"); */
								/*gdImageDestroy(im); */
								/*return 0; */
								im->tpixels[y][x] = 0;
							}
						} else {
							int ch;
							if (!gdGetByte (&ch, in)) {
								/*printf("EOF while reading\n"); */
								/*gdImageDestroy(im); */
								/*return 0; */
								ch = 0;
							}
							im->pixels[y][x] = ch;
						}
					}
				} else {
					for (x = xlo; x < xhi; x++) {
						if (im->trueColor) {
							/* 2.0.1: work around a gcc bug by being verbose.
							   TBB */
							int a = chunkBuf[chunkPos++] << 24;
							int r = chunkBuf[chunkPos++] << 16;
							int g = chunkBuf[chunkPos++] << 8;
							int b = chunkBuf[chunkPos++];
							/* 2.0.11: tpixels */
							im->tpixels[y][x] = a + r + g + b;
						} else {
							im->pixels[y][x] = chunkBuf[chunkPos++];
						}
					};
				};
				/*GD2_DBG(printf("\n")); */
			};
			chunkNum++;
		};
	};

	GD2_DBG (printf ("Freeing memory\n"));

	gdFree (chunkBuf);
	gdFree (compBuf);
	gdFree (chunkIdx);

	GD2_DBG (printf ("Done\n"));

	return im;

fail:
	gdImageDestroy (im);
	if (chunkBuf) {
		gdFree (chunkBuf);
	}
	if (compBuf) {
		gdFree (compBuf);
	}
	if (chunkIdx) {
		gdFree (chunkIdx);
	}
	return 0;
}

Project: libgd

BGD_DECLARE(gdImagePtr) gdImageCreateFromGd2Part (FILE * inFile, int srcx, int srcy, int w, int h)
{
	gdImagePtr im;
	gdIOCtx *in = gdNewFileCtx (inFile);

	if (in == NULL) return NULL;
	im = gdImageCreateFromGd2PartCtx (in, srcx, srcy, w, h);

	in->gd_free (in);

	return im;
}

Project: libgd

BGD_DECLARE(gdImagePtr) gdImageCreateFromGd2PartPtr (int size, void *data, int srcx, int srcy, int w,
        int h)
{
	gdImagePtr im;
	gdIOCtx *in = gdNewDynamicCtxEx (size, data, 0);
	if(!in)
		return 0;
	im = gdImageCreateFromGd2PartCtx (in, srcx, srcy, w, h);
	in->gd_free (in);
	return im;
}

Project: libgd

BGD_DECLARE(gdImagePtr) gdImageCreateFromGd2PartCtx (gdIOCtx * in, int srcx, int srcy, int w, int h)
{
	int scx, scy, ecx, ecy, fsx, fsy;
	int nc, ncx, ncy, cs, cx, cy;
	int x, y, ylo, yhi, xlo, xhi;
	int dstart, dpos;
	int i;
	/* 2.0.12: unsigned is correct; fixes problems with color munging.
	   Thanks to Steven Brown. */
	unsigned int ch;
	int vers, fmt;
	t_chunk_info *chunkIdx = NULL;
	unsigned char *chunkBuf = NULL;
	int chunkNum;
	int chunkMax = 0;
	uLongf chunkLen;
	int chunkPos = 0;
	int compMax;
	char *compBuf = NULL;

	gdImagePtr im;

	/* */
	/* The next few lines are basically copied from gd2CreateFromFile */
	/* - we change the file size, so don't want to use the code directly. */
	/*   but we do need to know the file size. */
	/* */
	if (_gd2GetHeader (in, &fsx, &fsy, &cs, &vers, &fmt, &ncx, &ncy, &chunkIdx)
	        != 1) {
		goto fail1;
	}

	GD2_DBG (printf ("File size is %dx%d\n", fsx, fsy));

	/* This is the difference - make a file based on size of chunks. */
	if (gd2_truecolor (fmt)) {
		im = gdImageCreateTrueColor (w, h);
	} else {
		im = gdImageCreate (w, h);
	}
	if (im == NULL) {
		goto fail1;
	};

	if (!_gdGetColors (in, im, vers == 2)) {
		goto fail2;
	}
	GD2_DBG (printf ("Image palette completed: %d colours\n", im->colorsTotal));

	/* Process the header info */
	nc = ncx * ncy;

	if (gd2_compressed (fmt)) {
		/* Find the maximum compressed chunk size. */
		compMax = 0;
		for (i = 0; (i < nc); i++) {
			if (chunkIdx[i].size > compMax) {
				compMax = chunkIdx[i].size;
			};
		};
		compMax++;

		if (im->trueColor) {
			chunkMax = cs * cs * 4;
		} else {
			chunkMax = cs * cs;
		}
		chunkBuf = gdCalloc (chunkMax, 1);
		if (!chunkBuf) {
			goto fail2;
		}
		compBuf = gdCalloc (compMax, 1);
		if (!compBuf) {
			goto fail2;
		}

	};

	/*      Don't bother with this... */
	/*      if ( (ncx != sx / cs) || (ncy != sy / cs)) { */
	/*              goto fail2; */
	/*      }; */


	/* Work out start/end chunks */
	scx = srcx / cs;
	scy = srcy / cs;
	if (scx < 0) {
		scx = 0;
	};
	if (scy < 0) {
		scy = 0;
	};

	ecx = (srcx + w) / cs;
	ecy = (srcy + h) / cs;
	if (ecx >= ncx) {
		ecx = ncx - 1;
	};
	if (ecy >= ncy) {
		ecy = ncy - 1;
	};

	/* Remember file position of image data. */
	dstart = gdTell (in);
	GD2_DBG (printf ("Data starts at %d\n", dstart));

	/* Loop through the chunks. */
	for (cy = scy; (cy <= ecy); cy++) {

		ylo = cy * cs;
		yhi = ylo + cs;
		if (yhi > fsy) {
			yhi = fsy;
		};

		for (cx = scx; (cx <= ecx); cx++) {

			xlo = cx * cs;
			xhi = xlo + cs;
			if (xhi > fsx) {
				xhi = fsx;
			};

			GD2_DBG (printf
			         ("Processing Chunk (%d, %d), from %d to %d\n", cx, cy, ylo,
			          yhi));

			if (!gd2_compressed (fmt)) {
				GD2_DBG (printf ("Using raw format data\n"));
				if (im->trueColor) {
					dpos =
					    (cy * (cs * fsx) * 4 + cx * cs * (yhi - ylo) * 4) +
					    dstart;
				} else {
					dpos = cy * (cs * fsx) + cx * cs * (yhi - ylo) + dstart;
				}
				/* gd 2.0.11: gdSeek returns TRUE on success, not 0.
				   Longstanding bug. 01/16/03 */
				if (!gdSeek (in, dpos)) {
					gd_error("Seek error\n");
					goto fail2;
				};
				GD2_DBG (printf
				         ("Reading (%d, %d) from position %d\n", cx, cy,
				          dpos - dstart));
			} else {
				chunkNum = cx + cy * ncx;

				chunkLen = chunkMax;
				if (!_gd2ReadChunk (chunkIdx[chunkNum].offset,
				                    compBuf,
				                    chunkIdx[chunkNum].size,
				                    (char *) chunkBuf, &chunkLen, in)) {
					printf ("Error reading comproessed chunk\n");
					goto fail2;
				};
				chunkPos = 0;
				GD2_DBG (printf
				         ("Reading (%d, %d) from chunk %d\n", cx, cy,
				          chunkNum));
			};

			GD2_DBG (printf
			         ("   into (%d, %d) - (%d, %d)\n", xlo, ylo, xhi, yhi));
			for (y = ylo; (y < yhi); y++) {

				for (x = xlo; x < xhi; x++) {
					if (!gd2_compressed (fmt)) {
						if (im->trueColor) {
							if (!gdGetInt ((int *) &ch, in)) {
								ch = 0;
								/*printf("EOF while reading file\n"); */
								/*goto fail2; */
							}
						} else {
							ch = gdGetC (in);
							if ((int) ch == EOF) {
								ch = 0;
								/*printf("EOF while reading file\n"); */
								/*goto fail2; */
							}
						}
					} else {
						if (im->trueColor) {
							ch = chunkBuf[chunkPos++];
							ch = (ch << 8) + chunkBuf[chunkPos++];
							ch = (ch << 8) + chunkBuf[chunkPos++];
							ch = (ch << 8) + chunkBuf[chunkPos++];
						} else {
							ch = chunkBuf[chunkPos++];
						}
					};

					/* Only use a point that is in the image. */
					if ((x >= srcx) && (x < (srcx + w)) && (x < fsx) && (x >= 0)
					        && (y >= srcy) && (y < (srcy + h)) && (y < fsy)
					        && (y >= 0)) {
						/* 2.0.11: tpixels */
						if (im->trueColor) {
							im->tpixels[y - srcy][x - srcx] = ch;
						} else {
							im->pixels[y - srcy][x - srcx] = ch;
						}
					}
				};
			};
		};
	};

	gdFree (chunkBuf);
	gdFree (compBuf);
	gdFree (chunkIdx);

	return im;

fail2:
	gdImageDestroy (im);
fail1:
	if (chunkBuf) {
		gdFree (chunkBuf);
	}
	if (compBuf) {
		gdFree (compBuf);
	}
	if (chunkIdx) {
		gdFree (chunkIdx);
	}
	return 0;

}

Project: libgd

BGD_DECLARE(gdImagePtr) gdImageCreateFromGd2 (FILE * inFile)
{
	_noLibzError();
	return NULL;
}

Project: libgd

BGD_DECLARE(gdImagePtr) gdImageCreateFromGd2Ctx (gdIOCtxPtr in)
{
	_noLibzError();
	return NULL;
}

Project: libgd

BGD_DECLARE(gdImagePtr) gdImageCreateFromGd2Ptr (int size, void *data)
{
	_noLibzError();
	return NULL;
}

Project: libgd

BGD_DECLARE(gdImagePtr) gdImageCreateFromGd2PartCtx (gdIOCtx * in, int srcx, int srcy, int w, int h)
{
	_noLibzError();
	return NULL;
}

Project: libgd

BGD_DECLARE(gdImagePtr) gdImageCreateFromGd2PartPtr (int size, void *data, int srcx, int srcy, int w,
        int h)
{
	_noLibzError();
	return NULL;
}

Project: libgd

_gd2CreateFromFile (gdIOCtxPtr in, int *sx, int *sy,
                    int *cs, int *vers, int *fmt,
                    int *ncx, int *ncy, t_chunk_info ** cidx)
{
	gdImagePtr im;

	if (_gd2GetHeader (in, sx, sy, cs, vers, fmt, ncx, ncy, cidx) != 1) {
		GD2_DBG (printf ("Bad GD2 header\n"));
		goto fail1;
	}
	if (gd2_truecolor (*fmt)) {
		im = gdImageCreateTrueColor (*sx, *sy);
	} else {
		im = gdImageCreate (*sx, *sy);
	}
	if (im == NULL) {
		GD2_DBG (printf ("Could not create gdImage\n"));
		goto fail2;
	};

	if (!_gdGetColors (in, im, (*vers) == 2)) {
		GD2_DBG (printf ("Could not read color palette\n"));
		goto fail3;
	}
	GD2_DBG (printf ("Image palette completed: %d colours\n", im->colorsTotal));

	return im;

fail3:
	gdImageDestroy (im);
fail2:
	gdFree(*cidx);
fail1:
	return 0;

}

Project: libgd

_gd2PutHeader (gdImagePtr im, gdIOCtx * out, int cs, int fmt, int cx, int cy)
{
	int i;

	/* Send the gd2 id, to verify file format. */
	for (i = 0; i < 4; i++) {
		gdPutC ((unsigned char) (GD2_ID[i]), out);
	};

	/* */
	/* We put the version info first, so future versions can easily change header info. */
	/* */
	gdPutWord (GD2_VERS, out);
	gdPutWord (im->sx, out);
	gdPutWord (im->sy, out);
	gdPutWord (cs, out);
	gdPutWord (fmt, out);
	gdPutWord (cx, out);
	gdPutWord (cy, out);

}

Project: libgd

_gd2ReadChunk (int offset, char *compBuf, int compSize, char *chunkBuf,
               uLongf * chunkLen, gdIOCtx * in)
{
	int zerr;

	if (gdTell (in) != offset) {
		GD2_DBG (printf ("Positioning in file to %d\n", offset));
		gdSeek (in, offset);
	} else {
		GD2_DBG (printf ("Already Positioned in file to %d\n", offset));
	};

	/* Read and uncompress an entire chunk. */
	GD2_DBG (printf ("Reading file\n"));
	if (gdGetBuf (compBuf, compSize, in) != compSize) {
		return FALSE;
	};
	GD2_DBG (printf
	         ("Got %d bytes. Uncompressing into buffer of %d bytes\n", compSize,
	          *chunkLen));
	zerr =
	    uncompress ((unsigned char *) chunkBuf, chunkLen,
	                (unsigned char *) compBuf, compSize);
	if (zerr != Z_OK) {
		GD2_DBG (printf ("Error %d from uncompress\n", zerr));
		return FALSE;
	};
	GD2_DBG (printf ("Got chunk\n"));
	return TRUE;
}

Project: libgd

static void _noLibzError (void)
{
	gd_error("GD2 support is not available - no libz\n");
}

Project: libgd

BGD_DECLARE(void) gdImageGd2 (gdImagePtr im, FILE * outFile, int cs, int fmt)
{
	gdIOCtx *out = gdNewFileCtx (outFile);
	if (out == NULL) return;
	_gdImageGd2 (im, out, cs, fmt);
	out->gd_free (out);
}

Project: libgd

BGD_DECLARE(void *) gdImageGd2Ptr (gdImagePtr im, int cs, int fmt, int *size)
{
	void *rv;
	gdIOCtx *out = gdNewDynamicCtx (2048, NULL);
	if (out == NULL) return NULL;
	_gdImageGd2 (im, out, cs, fmt);
	rv = gdDPExtractData (out, size);
	out->gd_free (out);
	return rv;
}

Project: libgd

BGD_DECLARE(void *) gdImageGd2Ptr (gdImagePtr im, int cs, int fmt, int *size)
{
	_noLibzError();
}