Commit message: Fixed CVE-2018-8788

Thanks to Eyal Itkin from Check Point Software Technologies.
CVE: CVE-2018-8788
CWE: CWE-787
Vulnerability description: FreeRDP prior to version 2.0.0-rc4 contains an Out-Of-Bounds Write of up to 4 bytes in function nsc_rle_decode() that results in a memory corruption and possibly even a remote code execution.

Project: FreeRDP

static void nsc_decode(NSC_CONTEXT* context)
static BOOL nsc_decode(NSC_CONTEXT* context)
 {
 	UINT16 x;
 	UINT16 y;
	UINT16 rw;
	BYTE shift;
	BYTE* bmpdata;
	size_t pos = 0;

	if (!context)
		return FALSE;

	rw = ROUND_UP_TO(context->width, 8);
	shift = context->ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */
	bmpdata = context->BitmapData;

	if (!bmpdata)
		return FALSE;
 
 	for (y = 0; y < context->height; y++)
 	{
		const BYTE* yplane;
		const BYTE* coplane;
		const BYTE* cgplane;
		const BYTE* aplane = context->priv->PlaneBuffers[3] + y * context->width; /* A */

		if (context->ChromaSubsamplingLevel)
		{
			yplane = context->priv->PlaneBuffers[0] + y * rw; /* Y */
			coplane = context->priv->PlaneBuffers[1] + (y >> 1) * (rw >>
			          1); /* Co, supersampled */
			cgplane = context->priv->PlaneBuffers[2] + (y >> 1) * (rw >>
			          1); /* Cg, supersampled */
		}
		else
		{
			yplane = context->priv->PlaneBuffers[0] + y * context->width; /* Y */
			coplane = context->priv->PlaneBuffers[1] + y * context->width; /* Co */
			cgplane = context->priv->PlaneBuffers[2] + y * context->width; /* Cg */
		}

		for (x = 0; x < context->width; x++)
		{
			INT16 y_val = (INT16) * yplane;
			INT16 co_val = (INT16)(INT8)(*coplane << shift);
			INT16 cg_val = (INT16)(INT8)(*cgplane << shift);
 			INT16 r_val = y_val + co_val - cg_val;
 			INT16 g_val = y_val + cg_val;
 			INT16 b_val = y_val - co_val - cg_val;

			if (pos + 4 > context->BitmapDataLength)
				return FALSE;

			pos += 4;
 			*bmpdata++ = MINMAX(b_val, 0, 0xFF);
 			*bmpdata++ = MINMAX(g_val, 0, 0xFF);
 			*bmpdata++ = MINMAX(r_val, 0, 0xFF);
			*bmpdata++ = *aplane;
			yplane++;
			coplane += (context->ChromaSubsamplingLevel ? x % 2 : 1);
			cgplane += (context->ChromaSubsamplingLevel ? x % 2 : 1);
 			aplane++;
 		}
 	}

	return TRUE;
 }

Project: FreeRDP
File: nsc.c

BOOL nsc_process_message(NSC_CONTEXT* context, UINT16 bpp,
                         UINT32 width, UINT32 height,
                         const BYTE* data, UINT32 length,
                         BYTE* pDstData, UINT32 DstFormat,
                         UINT32 nDstStride,
                         UINT32 nXDst, UINT32 nYDst, UINT32 nWidth,
                         UINT32 nHeight, UINT32 flip)
{
	wStream* s;
	BOOL ret;
	s = Stream_New((BYTE*)data, length);

	if (!s)
		return FALSE;

	if (nDstStride == 0)
		nDstStride = nWidth * GetBytesPerPixel(DstFormat);

	switch (bpp)
	{
		case 32:
			context->format = PIXEL_FORMAT_BGRA32;
			break;

		case 24:
			context->format = PIXEL_FORMAT_BGR24;
			break;

		case 16:
			context->format = PIXEL_FORMAT_BGR16;
			break;

		case 8:
			context->format = PIXEL_FORMAT_RGB8;
			break;

		case 4:
			context->format = PIXEL_FORMAT_A4;
			break;

		default:
			Stream_Free(s, TRUE);
			return FALSE;
	}

	context->width = width;
	context->height = height;
	ret = nsc_context_initialize(context, s);
	Stream_Free(s, FALSE);

	if (!ret)
 		return FALSE;
 
 	/* RLE decode */
	{
		BOOL rc;
		PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data)
		rc = nsc_rle_decompress_data(context);
		PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data)

		if (!rc)
			return FALSE;
	}
 	/* Colorloss recover, Chroma supersample and AYCoCg to ARGB Conversion in one step */
	{
		BOOL rc;
		PROFILER_ENTER(context->priv->prof_nsc_decode)
		rc = context->decode(context);
		PROFILER_EXIT(context->priv->prof_nsc_decode)

		if (!rc)
			return FALSE;
	}
 
 	if (!freerdp_image_copy(pDstData, DstFormat, nDstStride, nXDst, nYDst,
 	                        width, height, context->BitmapData,
	                        PIXEL_FORMAT_BGRA32, 0, 0, 0, NULL, flip))
		return FALSE;

	return TRUE;
}

Project: FreeRDP

static void nsc_rle_decode(BYTE* in, BYTE* out, UINT32 originalSize)
static BOOL nsc_rle_decode(BYTE* in, BYTE* out, UINT32 outSize, UINT32 originalSize)
 {
 	UINT32 len;
 	UINT32 left;
	BYTE value;
	left = originalSize;

	while (left > 4)
	{
		value = *in++;
 
 		if (left == 5)
 		{
			if (outSize < 1)
				return FALSE;

			outSize--;
 			*out++ = value;
 			left--;
 		}
		else if (value == *in)
		{
			in++;

			if (*in < 0xFF)
			{
				len = (UINT32) * in++;
				len += 2;
			}
			else
			{
				in++;
				len = *((UINT32*) in);
 				in += 4;
 			}
 
			if (outSize < len)
				return FALSE;

			outSize -= len;
 			FillMemory(out, len, value);
 			out += len;
 			left -= len;
 		}
 		else
 		{
			if (outSize < 1)
				return FALSE;

			outSize--;
 			*out++ = value;
 			left--;
 		}
 	}
 
	if ((outSize < 4) || (left < 4))
		return FALSE;

	memcpy(out, in, 4);
	return TRUE;
 }

Project: FreeRDP

static void nsc_rle_decompress_data(NSC_CONTEXT* context)
static BOOL nsc_rle_decompress_data(NSC_CONTEXT* context)
 {
 	UINT16 i;
 	BYTE* rle;
 	UINT32 planeSize;
 	UINT32 originalSize;

	if (!context)
		return FALSE;

 	rle = context->Planes;
 
 	for (i = 0; i < 4; i++)
	{
		originalSize = context->OrgByteCount[i];
 		planeSize = context->PlaneByteCount[i];
 
 		if (planeSize == 0)
		{
			if (context->priv->PlaneBuffersLength < originalSize)
				return FALSE;

 			FillMemory(context->priv->PlaneBuffers[i], originalSize, 0xFF);
		}
 		else if (planeSize < originalSize)
		{
			if (!nsc_rle_decode(rle, context->priv->PlaneBuffers[i], context->priv->PlaneBuffersLength,
			                    originalSize))
				return FALSE;
		}
 		else
		{
			if (context->priv->PlaneBuffersLength < originalSize)
				return FALSE;

 			CopyMemory(context->priv->PlaneBuffers[i], rle, originalSize);
		}
 
 		rle += planeSize;
 	}

	return TRUE;
 }

Project: FreeRDP

static void nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data,
static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data,
                                       UINT32 scanline)
 {
 	UINT16 x;
	UINT16 y;
	UINT16 rw;
	BYTE ccl;
	const BYTE* src;
	BYTE* yplane = NULL;
	BYTE* coplane = NULL;
	BYTE* cgplane = NULL;
	BYTE* aplane = NULL;
	INT16 r_val;
	INT16 g_val;
 	INT16 b_val;
 	BYTE a_val;
 	UINT32 tempWidth;

	if (!context || data || (scanline == 0))
		return FALSE;

 	tempWidth = ROUND_UP_TO(context->width, 8);
 	rw = (context->ChromaSubsamplingLevel ? tempWidth : context->width);
 	ccl = context->ColorLossLevel;
 
	if (context->priv->PlaneBuffersLength < rw * scanline)
		return FALSE;

	if (rw < scanline * 2)
		return FALSE;

 	for (y = 0; y < context->height; y++)
 	{
 		src = data + (context->height - 1 - y) * scanline;
		yplane = context->priv->PlaneBuffers[0] + y * rw;
		coplane = context->priv->PlaneBuffers[1] + y * rw;
		cgplane = context->priv->PlaneBuffers[2] + y * rw;
		aplane = context->priv->PlaneBuffers[3] + y * context->width;

		for (x = 0; x < context->width; x++)
		{
			switch (context->format)
			{
				case PIXEL_FORMAT_BGRX32:
					b_val = *src++;
					g_val = *src++;
					r_val = *src++;
					src++;
					a_val = 0xFF;
					break;

				case PIXEL_FORMAT_BGRA32:
					b_val = *src++;
					g_val = *src++;
					r_val = *src++;
					a_val = *src++;
					break;

				case PIXEL_FORMAT_RGBX32:
					r_val = *src++;
					g_val = *src++;
					b_val = *src++;
					src++;
					a_val = 0xFF;
					break;

				case PIXEL_FORMAT_RGBA32:
					r_val = *src++;
					g_val = *src++;
					b_val = *src++;
					a_val = *src++;
					break;

				case PIXEL_FORMAT_BGR24:
					b_val = *src++;
					g_val = *src++;
					r_val = *src++;
					a_val = 0xFF;
					break;

				case PIXEL_FORMAT_RGB24:
					r_val = *src++;
					g_val = *src++;
					b_val = *src++;
					a_val = 0xFF;
					break;

				case PIXEL_FORMAT_BGR16:
					b_val = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5));
					g_val = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3));
					r_val = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07));
					a_val = 0xFF;
					src += 2;
					break;

				case PIXEL_FORMAT_RGB16:
					r_val = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5));
					g_val = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3));
					b_val = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07));
					a_val = 0xFF;
					src += 2;
					break;

				case PIXEL_FORMAT_A4:
					{
						int shift;
						BYTE idx;
						shift = (7 - (x % 8));
						idx = ((*src) >> shift) & 1;
						idx |= (((*(src + 1)) >> shift) & 1) << 1;
						idx |= (((*(src + 2)) >> shift) & 1) << 2;
						idx |= (((*(src + 3)) >> shift) & 1) << 3;
						idx *= 3;
						r_val = (INT16) context->palette[idx];
						g_val = (INT16) context->palette[idx + 1];
						b_val = (INT16) context->palette[idx + 2];

						if (shift == 0)
							src += 4;
					}

					a_val = 0xFF;
					break;

				case PIXEL_FORMAT_RGB8:
					{
						int idx = (*src) * 3;
						r_val = (INT16) context->palette[idx];
						g_val = (INT16) context->palette[idx + 1];
						b_val = (INT16) context->palette[idx + 2];
						src++;
					}

					a_val = 0xFF;
					break;

				default:
					r_val = g_val = b_val = a_val = 0;
					break;
			}

			*yplane++ = (BYTE)((r_val >> 2) + (g_val >> 1) + (b_val >> 2));
			/* Perform color loss reduction here */
			*coplane++ = (BYTE)((r_val - b_val) >> ccl);
			*cgplane++ = (BYTE)((-(r_val >> 1) + g_val - (b_val >> 1)) >> ccl);
			*aplane++ = a_val;
		}

		if (context->ChromaSubsamplingLevel && (x % 2) == 1)
		{
			*yplane = *(yplane - 1);
			*coplane = *(coplane - 1);
			*cgplane = *(cgplane - 1);
		}
	}

	if (context->ChromaSubsamplingLevel && (y % 2) == 1)
	{
		yplane = context->priv->PlaneBuffers[0] + y * rw;
		coplane = context->priv->PlaneBuffers[1] + y * rw;
		cgplane = context->priv->PlaneBuffers[2] + y * rw;
		CopyMemory(yplane, yplane - rw, rw);
 		CopyMemory(coplane, coplane - rw, rw);
 		CopyMemory(cgplane, cgplane - rw, rw);
 	}

	return TRUE;
 }

Project: FreeRDP

static void nsc_encode_subsampling(NSC_CONTEXT* context)
static BOOL nsc_encode_subsampling(NSC_CONTEXT* context)
 {
 	UINT16 x;
 	UINT16 y;
 	UINT32 tempWidth;
 	UINT32 tempHeight;

	if (!context)
		return FALSE;

 	tempWidth = ROUND_UP_TO(context->width, 8);
 	tempHeight = ROUND_UP_TO(context->height, 2);
 
	if (tempHeight == 0)
		return FALSE;

	if (tempWidth > context->priv->PlaneBuffersLength / tempHeight)
		return FALSE;

 	for (y = 0; y < tempHeight >> 1; y++)
 	{
		BYTE* co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
		BYTE* cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
		const INT8* co_src0 = (INT8*) context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
		const INT8* co_src1 = co_src0 + tempWidth;
		const INT8* cg_src0 = (INT8*) context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
		const INT8* cg_src1 = cg_src0 + tempWidth;
 
 		for (x = 0; x < tempWidth >> 1; x++)
 		{
			*co_dst++ = (BYTE)(((INT16) * co_src0 + (INT16) * (co_src0 + 1) +
			                    (INT16) * co_src1 + (INT16) * (co_src1 + 1)) >> 2);
			*cg_dst++ = (BYTE)(((INT16) * cg_src0 + (INT16) * (cg_src0 + 1) +
			                    (INT16) * cg_src1 + (INT16) * (cg_src1 + 1)) >> 2);
			co_src0 += 2;
			co_src1 += 2;
			cg_src0 += 2;
 			cg_src1 += 2;
 		}
 	}

	return TRUE;
 }

Project: FreeRDP

static void nsc_encode_sse2(NSC_CONTEXT* context, const BYTE* data,
static BOOL nsc_encode_sse2(NSC_CONTEXT* context, const BYTE* data,
                             UINT32 scanline)
 {
 	nsc_encode_argb_to_aycocg_sse2(context, data, scanline);

	if (context->ChromaSubsamplingLevel > 0)
 	{
 		nsc_encode_subsampling_sse2(context);
 	}

	return TRUE;
 }