Commit message: gdImageScaleTwoPass memory leak fix

Fixing memory leak in gdImageScaleTwoPass, as reported by @cmb69 and
confirmed by @vapier.  This bug actually bit me in production and I'm
very thankful that it was reported with an easy fix.

Fixes #173.
CVE: CVE-2015-8877
CWE: CWE-399
Vulnerability description: The gdImageScaleTwoPass function in gd_interpolation.c in the GD Graphics Library (aka libgd) before 2.2.0, as used in PHP before 5.6.12, uses inconsistent allocate and free approaches, which allows remote attackers to cause a denial of service (memory consumption) via a crafted call, as demonstrated by a call to the PHP imagescale function.

Project: libgd
File: gd_interpolation.c

BGD_DECLARE(gdImagePtr) gdImageScale(const gdImagePtr src, const unsigned int new_width, const unsigned int new_height)
{
	gdImagePtr im_scaled = NULL;

	if (src == NULL || src->interpolation_id < 0 || src->interpolation_id > GD_METHOD_COUNT) {
		return 0;
	}

	switch (src->interpolation_id) {
		/*Special cases, optimized implementations */
		case GD_NEAREST_NEIGHBOUR:
			im_scaled = gdImageScaleNearestNeighbour(src, new_width, new_height);
			break;

		case GD_BILINEAR_FIXED:
			im_scaled = gdImageScaleBilinear(src, new_width, new_height);
			break;

		case GD_BICUBIC_FIXED:
			im_scaled = gdImageScaleBicubicFixed(src, new_width, new_height);
			break;

		/* generic */
		default:
			if (src->interpolation == NULL) {
				return NULL;
			}
			im_scaled = gdImageScaleTwoPass(src, new_width, new_height);
			break;
	}

	return im_scaled;
}

Project: libgd
File: gd_interpolation.c

void gdDumpRect(const char *msg, gdRectPtr r)
{
	printf("%s (%i, %i) (%i, %i)\n", msg, r->x, r->y, r->width, r->height);
}

Project: libgd
File: gd_interpolation.c

gdImageRotateBilinear(gdImagePtr src, const float degrees, const int bgColor)
{
	float _angle = (float)((- degrees / 180.0f) * M_PI);
	const unsigned int src_w = gdImageSX(src);
	const unsigned int src_h = gdImageSY(src);
	unsigned int new_width = abs((int)(src_w*cos(_angle))) + abs((int)(src_h*sin(_angle) + 0.5f));
	unsigned int new_height = abs((int)(src_w*sin(_angle))) + abs((int)(src_h*cos(_angle) + 0.5f));
	const gdFixed f_0_5 = gd_ftofx(0.5f);
	const gdFixed f_H = gd_itofx(src_h/2);
	const gdFixed f_W = gd_itofx(src_w/2);
	const gdFixed f_cos = gd_ftofx(cos(-_angle));
	const gdFixed f_sin = gd_ftofx(sin(-_angle));
	const gdFixed f_1 = gd_itofx(1);
	unsigned int i;
	unsigned int dst_offset_x;
	unsigned int dst_offset_y = 0;
	unsigned int src_offset_x, src_offset_y;
	gdImagePtr dst;

	/* impact perf a bit, but not that much. Implementation for palette
	   images can be done at a later point.
	*/
	if (src->trueColor == 0) {
		gdImagePaletteToTrueColor(src);
	}

	dst = gdImageCreateTrueColor(new_width, new_height);
	if (dst == NULL) {
		return NULL;
	}
	dst->saveAlphaFlag = 1;

	for (i = 0; i < new_height; i++) {
		unsigned int j;
		dst_offset_x = 0;

		for (j=0; j < new_width; j++) {
			const gdFixed f_i = gd_itofx((int)i - (int)new_height / 2);
			const gdFixed f_j = gd_itofx((int)j - (int)new_width  / 2);
			const gdFixed f_m = gd_mulfx(f_j,f_sin) + gd_mulfx(f_i,f_cos) + f_0_5 + f_H;
			const gdFixed f_n = gd_mulfx(f_j,f_cos) - gd_mulfx(f_i,f_sin) + f_0_5 + f_W;
			const unsigned int m = gd_fxtoi(f_m);
			const unsigned int n = gd_fxtoi(f_n);

			if ((m > 0) && (m < src_h - 1) && (n > 0) && (n < src_w - 1)) {
				const gdFixed f_f = f_m - gd_itofx(m);
				const gdFixed f_g = f_n - gd_itofx(n);
				const gdFixed f_w1 = gd_mulfx(f_1-f_f, f_1-f_g);
				const gdFixed f_w2 = gd_mulfx(f_1-f_f, f_g);
				const gdFixed f_w3 = gd_mulfx(f_f, f_1-f_g);
				const gdFixed f_w4 = gd_mulfx(f_f, f_g);

				if (n < src_w - 1) {
					src_offset_x = n + 1;
					src_offset_y = m;
				}

				if (m < src_h - 1) {
					src_offset_x = n;
					src_offset_y = m + 1;
				}

				if (!((n >= src_w - 1) || (m >= src_h - 1))) {
					src_offset_x = n + 1;
					src_offset_y = m + 1;
				}
				{
					const int pixel1 = src->tpixels[src_offset_y][src_offset_x];
					register int pixel2, pixel3, pixel4;

					if (src_offset_y + 1 >= src_h) {
						pixel2 = bgColor;
						pixel3 = bgColor;
						pixel4 = bgColor;
					} else if (src_offset_x + 1 >= src_w) {
						pixel2 = bgColor;
						pixel3 = bgColor;
						pixel4 = bgColor;
					} else {
					    pixel2 = src->tpixels[src_offset_y][src_offset_x + 1];
						pixel3 = src->tpixels[src_offset_y + 1][src_offset_x];
						pixel4 = src->tpixels[src_offset_y + 1][src_offset_x + 1];
					}
					{
						const gdFixed f_r1 = gd_itofx(gdTrueColorGetRed(pixel1));
						const gdFixed f_r2 = gd_itofx(gdTrueColorGetRed(pixel2));
						const gdFixed f_r3 = gd_itofx(gdTrueColorGetRed(pixel3));
						const gdFixed f_r4 = gd_itofx(gdTrueColorGetRed(pixel4));
						const gdFixed f_g1 = gd_itofx(gdTrueColorGetGreen(pixel1));
						const gdFixed f_g2 = gd_itofx(gdTrueColorGetGreen(pixel2));
						const gdFixed f_g3 = gd_itofx(gdTrueColorGetGreen(pixel3));
						const gdFixed f_g4 = gd_itofx(gdTrueColorGetGreen(pixel4));
						const gdFixed f_b1 = gd_itofx(gdTrueColorGetBlue(pixel1));
						const gdFixed f_b2 = gd_itofx(gdTrueColorGetBlue(pixel2));
						const gdFixed f_b3 = gd_itofx(gdTrueColorGetBlue(pixel3));
						const gdFixed f_b4 = gd_itofx(gdTrueColorGetBlue(pixel4));
						const gdFixed f_a1 = gd_itofx(gdTrueColorGetAlpha(pixel1));
						const gdFixed f_a2 = gd_itofx(gdTrueColorGetAlpha(pixel2));
						const gdFixed f_a3 = gd_itofx(gdTrueColorGetAlpha(pixel3));
						const gdFixed f_a4 = gd_itofx(gdTrueColorGetAlpha(pixel4));
						const gdFixed f_red = gd_mulfx(f_w1, f_r1) + gd_mulfx(f_w2, f_r2) + gd_mulfx(f_w3, f_r3) + gd_mulfx(f_w4, f_r4);
						const gdFixed f_green = gd_mulfx(f_w1, f_g1) + gd_mulfx(f_w2, f_g2) + gd_mulfx(f_w3, f_g3) + gd_mulfx(f_w4, f_g4);
						const gdFixed f_blue = gd_mulfx(f_w1, f_b1) + gd_mulfx(f_w2, f_b2) + gd_mulfx(f_w3, f_b3) + gd_mulfx(f_w4, f_b4);
						const gdFixed f_alpha = gd_mulfx(f_w1, f_a1) + gd_mulfx(f_w2, f_a2) + gd_mulfx(f_w3, f_a3) + gd_mulfx(f_w4, f_a4);

						const unsigned char red   = (unsigned char) CLAMP(gd_fxtoi(f_red),   0, 255);
						const unsigned char green = (unsigned char) CLAMP(gd_fxtoi(f_green), 0, 255);
						const unsigned char blue  = (unsigned char) CLAMP(gd_fxtoi(f_blue),  0, 255);
						const unsigned char alpha = (unsigned char) CLAMP(gd_fxtoi(f_alpha), 0, 127);

						dst->tpixels[dst_offset_y][dst_offset_x++] = gdTrueColorAlpha(red, green, blue, alpha);
					}
				}
			} else {
				dst->tpixels[dst_offset_y][dst_offset_x++] = bgColor;
			}
		}
		dst_offset_y++;
	}
	return dst;
}

Project: libgd
File: gd_interpolation.c

BGD_DECLARE(int) gdTransformAffineGetImage(gdImagePtr *dst,
		  const gdImagePtr src,
		  gdRectPtr src_area,
		  const double affine[6])
{
	int res;
	double m[6];
	gdRect bbox;
	gdRect area_full;

	if (src_area == NULL) {
		area_full.x = 0;
		area_full.y = 0;
		area_full.width  = gdImageSX(src);
		area_full.height = gdImageSY(src);
		src_area = &area_full;
	}

	gdTransformAffineBoundingBox(src_area, affine, &bbox);

	*dst = gdImageCreateTrueColor(bbox.width, bbox.height);
	if (*dst == NULL) {
		return GD_FALSE;
	}
	(*dst)->saveAlphaFlag = 1;

	if (!src->trueColor) {
		gdImagePaletteToTrueColor(src);
	}
	
	/* Translate to dst origin (0,0) */
	gdAffineTranslate(m, -bbox.x, -bbox.y);
	gdAffineConcat(m, affine, m);

	gdImageAlphaBlending(*dst, 0);

	res = gdTransformAffineCopy(*dst,
		  0,0,
		  src,
		  src_area,
		  m);

	if (res != GD_TRUE) {
		gdImageDestroy(*dst);
		dst = NULL;
		return GD_FALSE;
	} else {
		return GD_TRUE;
	}
}