Commit message: Fixed int overflow in PresentationContext_new

Thanks to hac425 CVE-2020-11038
CVE: CVE-2020-11038
CWE: CWE-190
Vulnerability description: In FreeRDP less than or equal to 2.0.0, an Integer Overflow to Buffer Overflow exists. When using /video redirection, a manipulated server can instruct the client to allocate a buffer with a smaller size than requested due to an integer overflow in size calculation. With later messages, the server can manipulate the client to write data out of bound to the previously allocated buffer. This has been patched in 2.1.0.

Project: FreeRDP
File: video_main.c

static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
                                                    UINT32 x, UINT32 y, UINT32 width, UINT32 height)
{
	VideoClientContextPriv* priv = video->priv;
	PresentationContext* ret = calloc(1, sizeof(*ret));
	if (!ret)
		return NULL;

	ret->video = video;
	ret->PresentationId = PresentationId;

	ret->h264 = h264_context_new(FALSE);
	if (!ret->h264)
	{
		WLog_ERR(TAG, "unable to create a h264 context");
		goto error_h264;
	}
	h264_context_reset(ret->h264, width, height);

	ret->currentSample = Stream_New(NULL, 4096);
	if (!ret->currentSample)
	{
		WLog_ERR(TAG, "unable to create current packet stream");
		goto error_currentSample;
	}

	ret->surfaceData = BufferPool_Take(priv->surfacePool, width * height * 4);
	if (!ret->surfaceData)
	{
		WLog_ERR(TAG, "unable to allocate surfaceData");
		goto error_surfaceData;
	}

	ret->surface = video->createSurface(video, ret->surfaceData, x, y, width, height);
	if (!ret->surface)
	{
		WLog_ERR(TAG, "unable to create surface");
		goto error_surface;
	}

	ret->yuv = yuv_context_new(FALSE);
	if (!ret->yuv)
	{
		WLog_ERR(TAG, "unable to create YUV decoder");
		goto error_yuv;
	}

	yuv_context_reset(ret->yuv, width, height);
	ret->refCounter = 1;
	return ret;

error_yuv:
	video->deleteSurface(video, ret->surface);
error_surface:
	BufferPool_Return(priv->surfacePool, ret->surfaceData);
error_surfaceData:
	Stream_Free(ret->currentSample, TRUE);
error_currentSample:
	h264_context_free(ret->h264);
error_h264:
	free(ret);
	return NULL;
}

Project: FreeRDP
File: video_main.c

static PresentationContext* PresentationContext_new(VideoClientContext* video, BYTE PresentationId,
                                                    UINT32 x, UINT32 y, UINT32 width, UINT32 height)
{
	size_t s;
	VideoClientContextPriv* priv = video->priv;
	PresentationContext* ret;
	s = width * height * 4ULL;
	if (s > INT32_MAX)
		return NULL;

	ret = calloc(1, sizeof(*ret));
	if (!ret)
		return NULL;

	ret->video = video;
	ret->PresentationId = PresentationId;

	ret->h264 = h264_context_new(FALSE);
	if (!ret->h264)
	{
		WLog_ERR(TAG, "unable to create a h264 context");
		goto error_h264;
	}
	h264_context_reset(ret->h264, width, height);

	ret->currentSample = Stream_New(NULL, 4096);
	if (!ret->currentSample)
	{
		WLog_ERR(TAG, "unable to create current packet stream");
		goto error_currentSample;
	}

	ret->surfaceData = BufferPool_Take(priv->surfacePool, s);
	if (!ret->surfaceData)
	{
		WLog_ERR(TAG, "unable to allocate surfaceData");
		goto error_surfaceData;
	}

	ret->surface = video->createSurface(video, ret->surfaceData, x, y, width, height);
	if (!ret->surface)
	{
		WLog_ERR(TAG, "unable to create surface");
		goto error_surface;
	}

	ret->yuv = yuv_context_new(FALSE);
	if (!ret->yuv)
	{
		WLog_ERR(TAG, "unable to create YUV decoder");
		goto error_yuv;
	}

	yuv_context_reset(ret->yuv, width, height);
	ret->refCounter = 1;
	return ret;

error_yuv:
	video->deleteSurface(video, ret->surface);
error_surface:
	BufferPool_Return(priv->surfacePool, ret->surfaceData);
error_surfaceData:
	Stream_Free(ret->currentSample, TRUE);
error_currentSample:
	h264_context_free(ret->h264);
error_h264:
	free(ret);
	return NULL;
}