Commit message: Fix invalid read in gdImageCreateFromTiffPtr()

tiff_invalid_read.tiff is corrupt, and causes an invalid read in
gdImageCreateFromTiffPtr(), but not in gdImageCreateFromTiff(). The culprit
is dynamicGetbuf(), which doesn't check for out-of-bound reads. In this case,
dynamicGetbuf() is called with a negative dp->pos, but also positive buffer
overflows have to be handled, in which case 0 has to be returned (cf. commit
75e29a9).

Fixing dynamicGetbuf() exhibits that the corrupt TIFF would still create
the image, because the return value of TIFFReadRGBAImage() is not checked.
We do that, and let createFromTiffRgba() fail if TIFFReadRGBAImage() fails.

This issue had been reported by Ibrahim El-Sayed to security@libgd.org.

CVE-2016-6911
CVE: CVE-2016-6911
CWE: CWE-125
Vulnerability description: The dynamicGetbuf function in the GD Graphics Library (aka libgd) before 2.2.4 allows remote attackers to cause a denial of service (out-of-bounds read) via a crafted TIFF image.

Project: libgd

BGD_DECLARE(void) gdImageTiffCtx(gdImagePtr image, gdIOCtx *out)
{
	int clipx1P, clipy1P, clipx2P, clipy2P;
	int bitDepth = 24;

	/* First, switch off clipping, or we'll not get all the image! */
	gdImageGetClip(image, &clipx1P, &clipy1P, &clipx2P, &clipy2P);

	/* use the appropriate routine depending on the bit depth of the image */
	if(image->trueColor) {
		bitDepth = 24;
	} else if(image->colorsTotal == 2) {
		bitDepth = 1;
	} else {
		bitDepth = 8;
	}

	tiffWriter(image, out, bitDepth);

	/* reset clipping area to the gd image's original values */
	gdImageSetClip(image, clipx1P, clipy1P, clipx2P, clipy2P);
}

Project: libgd

static void readTiff8bit (const unsigned char *src,
                          gdImagePtr im,
                          uint16       photometric,
                          int          startx,
                          int          starty,
                          int          width,
                          int          height,
                          char         has_alpha,
                          int          extra,
                          int          align)
{
	int    red, green, blue, alpha;
	int    x, y;

	(void)extra;
	(void)align;

	switch (photometric) {
	case PHOTOMETRIC_PALETTE:
		/* Palette has no alpha (see TIFF specs for more details */
		for (y = starty; y < starty + height; y++) {
			for (x = startx; x < startx + width; x++) {
				gdImageSetPixel(im, x, y,*(src++));
			}
		}
		break;

	case PHOTOMETRIC_RGB:
		if (has_alpha) {
			gdImageAlphaBlending(im, 0);
			gdImageSaveAlpha(im, 1);

			for (y = starty; y < starty + height; y++) {
				for (x = startx; x < startx + width; x++) {
					red   = *src++;
					green = *src++;
					blue  = *src++;
					alpha = *src++;
					red   = MIN (red, alpha);
					blue  = MIN (blue, alpha);
					green = MIN (green, alpha);

					if (alpha) {
						gdImageSetPixel(im, x, y, gdTrueColorAlpha(red * 255 / alpha, green * 255 / alpha, blue * 255 /alpha, gdAlphaMax - (alpha >> 1)));
					} else {
						gdImageSetPixel(im, x, y, gdTrueColorAlpha(red, green, blue, gdAlphaMax - (alpha >> 1)));
					}
				}
			}

		} else {
			for (y = 0; y < height; y++) {
				for (x = 0; x < width; x++) {
					register unsigned char r = *src++;
					register unsigned char g = *src++;
					register unsigned char b = *src++;

					gdImageSetPixel(im, x, y, gdTrueColor(r, g, b));
				}
			}
		}
		break;

	case PHOTOMETRIC_MINISWHITE:
		if (has_alpha) {
			/* We don't process the extra yet */
		} else {
			for (y = starty; y < starty + height; y++) {
				for (x = startx; x < startx + width; x++) {
					gdImageSetPixel(im, x, y, ~(*src++));
				}
			}
		}
		break;

	case PHOTOMETRIC_MINISBLACK:
		if (has_alpha) {
			/* We don't process the extra yet */
		} else {
			for (y = starty; y < height; y++) {
				for (x = 0; x < width; x++) {
					gdImageSetPixel(im, x, y, *src++);
				}
			}
		}
		break;
	}
}