Commit message: Fixed CVE-2018-8788

Thanks to Eyal Itkin from Check Point Software Technologies.
CVE: CVE-2018-8788
CWE: CWE-787
Vulnerability description: FreeRDP prior to version 2.0.0-rc4 contains an Out-Of-Bounds Write of up to 4 bytes in function nsc_rle_decode() that results in a memory corruption and possibly even a remote code execution.

Project: FreeRDP
File: nsc.c

void nsc_context_free(NSC_CONTEXT* context)
{
	size_t i;

	if (!context)
		return;

	if (context->priv)
	{
		for (i = 0; i < 4; i++)
			free(context->priv->PlaneBuffers[i]);

		BufferPool_Free(context->priv->PlanePool);
		nsc_profiler_print(context->priv);
		PROFILER_FREE(context->priv->prof_nsc_rle_decompress_data)
		PROFILER_FREE(context->priv->prof_nsc_decode)
		PROFILER_FREE(context->priv->prof_nsc_rle_compress_data)
		PROFILER_FREE(context->priv->prof_nsc_encode)
		free(context->priv);
	}

	free(context->BitmapData);
	free(context);
}

Project: FreeRDP
File: nsc.c

static BOOL nsc_context_initialize(NSC_CONTEXT* context, wStream* s)
{
	int i;
	UINT32 length;
	UINT32 tempWidth;
	UINT32 tempHeight;

	if (!nsc_stream_initialize(context, s))
		return FALSE;

	length = context->width * context->height * 4;

	if (!context->BitmapData)
	{
		context->BitmapData = calloc(1, length + 16);

		if (!context->BitmapData)
			return FALSE;

		context->BitmapDataLength = length;
	}
	else if (length > context->BitmapDataLength)
	{
		void* tmp;
		tmp = realloc(context->BitmapData, length + 16);

		if (!tmp)
			return FALSE;

		context->BitmapData = tmp;
		context->BitmapDataLength = length;
	}

	tempWidth = ROUND_UP_TO(context->width, 8);
	tempHeight = ROUND_UP_TO(context->height, 2);
	/* The maximum length a decoded plane can reach in all cases */
	length = tempWidth * tempHeight;

	if (length > context->priv->PlaneBuffersLength)
	{
		for (i = 0; i < 4; i++)
		{
			void* tmp = (BYTE*) realloc(context->priv->PlaneBuffers[i], length);

			if (!tmp)
				return FALSE;

			context->priv->PlaneBuffers[i] = tmp;
		}

		context->priv->PlaneBuffersLength = length;
	}

	for (i = 0; i < 4; i++)
	{
		context->OrgByteCount[i] = context->width * context->height;
	}

	if (context->ChromaSubsamplingLevel)
	{
		context->OrgByteCount[0] = tempWidth * context->height;
		context->OrgByteCount[1] = (tempWidth >> 1) * (tempHeight >> 1);
		context->OrgByteCount[2] = context->OrgByteCount[1];
	}

	return TRUE;
}

Project: FreeRDP
File: nsc.c

NSC_CONTEXT* nsc_context_new(void)
{
	NSC_CONTEXT* context;
	context = (NSC_CONTEXT*) calloc(1, sizeof(NSC_CONTEXT));

	if (!context)
		return NULL;

	context->priv = (NSC_CONTEXT_PRIV*) calloc(1, sizeof(NSC_CONTEXT_PRIV));

	if (!context->priv)
		goto error;

	context->priv->log = WLog_Get("com.freerdp.codec.nsc");
	WLog_OpenAppender(context->priv->log);
	context->BitmapData = NULL;
	context->decode = nsc_decode;
	context->encode = nsc_encode;
	context->priv->PlanePool = BufferPool_New(TRUE, 0, 16);

	if (!context->priv->PlanePool)
		goto error;

	PROFILER_CREATE(context->priv->prof_nsc_rle_decompress_data,
	                "nsc_rle_decompress_data")
	PROFILER_CREATE(context->priv->prof_nsc_decode, "nsc_decode")
	PROFILER_CREATE(context->priv->prof_nsc_rle_compress_data,
	                "nsc_rle_compress_data")
	PROFILER_CREATE(context->priv->prof_nsc_encode, "nsc_encode")
	/* Default encoding parameters */
	context->ColorLossLevel = 3;
	context->ChromaSubsamplingLevel = 1;
	/* init optimized methods */
	NSC_INIT_SIMD(context);
	return context;
error:
	nsc_context_free(context);
	return NULL;
}

Project: FreeRDP
File: nsc.c

BOOL nsc_context_reset(NSC_CONTEXT* context, UINT32 width, UINT32 height)
{
	if (!context)
		return FALSE;

	context->width = width;
	context->height = height;
	return TRUE;
}

Project: FreeRDP
File: nsc.c

BOOL nsc_context_set_pixel_format(NSC_CONTEXT* context, UINT32 pixel_format)
{
	if (!context)
		return FALSE;

	context->format = pixel_format;
	return TRUE;
}

Project: FreeRDP
File: nsc.c

static void nsc_profiler_print(NSC_CONTEXT_PRIV* priv)
{
	PROFILER_PRINT_HEADER
	PROFILER_PRINT(priv->prof_nsc_rle_decompress_data)
	PROFILER_PRINT(priv->prof_nsc_decode)
	PROFILER_PRINT(priv->prof_nsc_rle_compress_data)
	PROFILER_PRINT(priv->prof_nsc_encode)
	PROFILER_PRINT_FOOTER
}

Project: FreeRDP
File: nsc.c

 static BOOL nsc_stream_initialize(NSC_CONTEXT* context, wStream* s)
{
	int i;

	if (Stream_GetRemainingLength(s) < 20)
		return FALSE;

	for (i = 0; i < 4; i++)
		Stream_Read_UINT32(s, context->PlaneByteCount[i]);

	Stream_Read_UINT8(s, context->ColorLossLevel); /* ColorLossLevel (1 byte) */
	Stream_Read_UINT8(s,
	                  context->ChromaSubsamplingLevel); /* ChromaSubsamplingLevel (1 byte) */
	Stream_Seek(s, 2); /* Reserved (2 bytes) */
	context->Planes = Stream_Pointer(s);
	return TRUE;
}

Project: FreeRDP
File: nsc_encode.c

BOOL nsc_compose_message(NSC_CONTEXT* context, wStream* s, const BYTE* data,
                         UINT32 width, UINT32 height, UINT32 scanline)
{
	NSC_MESSAGE s_message = { 0 };
	NSC_MESSAGE* message = &s_message;
	context->width = width;
	context->height = height;

	if (!nsc_context_initialize_encode(context))
		return FALSE;

	/* ARGB to AYCoCg conversion, chroma subsampling and colorloss reduction */
	PROFILER_ENTER(context->priv->prof_nsc_encode)
	context->encode(context, data, scanline);
	PROFILER_EXIT(context->priv->prof_nsc_encode)
	/* RLE encode */
	PROFILER_ENTER(context->priv->prof_nsc_rle_compress_data)
	nsc_rle_compress_data(context);
	PROFILER_EXIT(context->priv->prof_nsc_rle_compress_data)
	message->PlaneBuffers[0] = context->priv->PlaneBuffers[0];
	message->PlaneBuffers[1] = context->priv->PlaneBuffers[1];
	message->PlaneBuffers[2] = context->priv->PlaneBuffers[2];
	message->PlaneBuffers[3] = context->priv->PlaneBuffers[3];
	message->LumaPlaneByteCount = context->PlaneByteCount[0];
	message->OrangeChromaPlaneByteCount = context->PlaneByteCount[1];
	message->GreenChromaPlaneByteCount = context->PlaneByteCount[2];
	message->AlphaPlaneByteCount = context->PlaneByteCount[3];
	message->ColorLossLevel = context->ColorLossLevel;
	message->ChromaSubsamplingLevel = context->ChromaSubsamplingLevel;
	return nsc_write_message(context, s, message);
}

Project: FreeRDP
File: nsc_encode.c

UINT32 nsc_compute_byte_count(NSC_CONTEXT* context, UINT32* ByteCount,
                              UINT32 width, UINT32 height)
{
	UINT32 tempWidth;
	UINT32 tempHeight;
	UINT32 maxPlaneSize;
	tempWidth = ROUND_UP_TO(width, 8);
	tempHeight = ROUND_UP_TO(height, 2);
	maxPlaneSize = tempWidth * tempHeight + 16;

	if (context->ChromaSubsamplingLevel)
	{
		ByteCount[0] = tempWidth * height;
		ByteCount[1] = tempWidth * tempHeight / 4;
		ByteCount[2] = tempWidth * tempHeight / 4;
		ByteCount[3] = width * height;
	}
	else
	{
		ByteCount[0] = width * height;
		ByteCount[1] = width * height;
		ByteCount[2] = width * height;
		ByteCount[3] = width * height;
	}

	return maxPlaneSize;
}

Project: FreeRDP
File: nsc_encode.c

NSC_MESSAGE* nsc_encode_messages(NSC_CONTEXT* context, const BYTE* data,
                                 UINT32 x, UINT32 y, UINT32 width, UINT32 height,
                                 UINT32 scanline, UINT32* numMessages,
                                 UINT32 maxDataSize)
{
	UINT32 i, j, k;
	UINT32 dataOffset;
	UINT32 rows, cols;
	UINT32 BytesPerPixel;
	UINT32 MaxRegionWidth;
	UINT32 MaxRegionHeight;
	UINT32 ByteCount[4];
	UINT32 MaxPlaneSize;
	UINT32 MaxMessageSize;
	NSC_MESSAGE* messages;
	UINT32 PaddedMaxPlaneSize;
	k = 0;
	MaxRegionWidth = 64 * 4;
	MaxRegionHeight = 64 * 2;
	BytesPerPixel = GetBytesPerPixel(context->format);
	rows = (width + (MaxRegionWidth - (width % MaxRegionWidth))) / MaxRegionWidth;
	cols = (height + (MaxRegionHeight - (height % MaxRegionHeight))) /
	       MaxRegionHeight;
	*numMessages = rows * cols;
	MaxPlaneSize = nsc_compute_byte_count(context, (UINT32*) ByteCount, width,
	                                      height);
	MaxMessageSize = ByteCount[0] + ByteCount[1] + ByteCount[2] + ByteCount[3] + 20;
	maxDataSize -= 1024; /* reserve enough space for headers */
	messages = (NSC_MESSAGE*) calloc(*numMessages, sizeof(NSC_MESSAGE));

	if (!messages)
		return NULL;

	for (i = 0; i < rows; i++)
	{
		for (j = 0; j < cols; j++)
		{
			messages[k].x = x + (i * MaxRegionWidth);
			messages[k].y = y + (j * MaxRegionHeight);
			messages[k].width = (i < (rows - 1)) ? MaxRegionWidth : width -
			                    (i * MaxRegionWidth);
			messages[k].height = (j < (cols - 1)) ? MaxRegionHeight : height -
			                     (j * MaxRegionHeight);
			messages[k].data = data;
			messages[k].scanline = scanline;
			messages[k].MaxPlaneSize = nsc_compute_byte_count(context,
			                           (UINT32*) messages[k].OrgByteCount, messages[k].width, messages[k].height);
			k++;
		}
	}

	*numMessages = k;

	for (i = 0; i < *numMessages; i++)
	{
		PaddedMaxPlaneSize = messages[i].MaxPlaneSize + 32;
		messages[i].PlaneBuffer = (BYTE*) BufferPool_Take(context->priv->PlanePool,
		                          PaddedMaxPlaneSize * 5);

		if (!messages[i].PlaneBuffer)
			goto fail;

		messages[i].PlaneBuffers[0] = (BYTE*) &
		                              (messages[i].PlaneBuffer[(PaddedMaxPlaneSize * 0) + 16]);
		messages[i].PlaneBuffers[1] = (BYTE*) &
		                              (messages[i].PlaneBuffer[(PaddedMaxPlaneSize * 1) + 16]);
		messages[i].PlaneBuffers[2] = (BYTE*) &
		                              (messages[i].PlaneBuffer[(PaddedMaxPlaneSize * 2) + 16]);
		messages[i].PlaneBuffers[3] = (BYTE*) &
		                              (messages[i].PlaneBuffer[(PaddedMaxPlaneSize * 3) + 16]);
		messages[i].PlaneBuffers[4] = (BYTE*) &
		                              (messages[i].PlaneBuffer[(PaddedMaxPlaneSize * 4) + 16]);
	}

	for (i = 0; i < *numMessages; i++)
	{
		context->width = messages[i].width;
		context->height = messages[i].height;
		context->OrgByteCount[0] = messages[i].OrgByteCount[0];
		context->OrgByteCount[1] = messages[i].OrgByteCount[1];
		context->OrgByteCount[2] = messages[i].OrgByteCount[2];
		context->OrgByteCount[3] = messages[i].OrgByteCount[3];
		context->priv->PlaneBuffersLength = messages[i].MaxPlaneSize;
		context->priv->PlaneBuffers[0] = messages[i].PlaneBuffers[0];
		context->priv->PlaneBuffers[1] = messages[i].PlaneBuffers[1];
		context->priv->PlaneBuffers[2] = messages[i].PlaneBuffers[2];
		context->priv->PlaneBuffers[3] = messages[i].PlaneBuffers[3];
		context->priv->PlaneBuffers[4] = messages[i].PlaneBuffers[4];
		dataOffset = (messages[i].y * messages[i].scanline) + (messages[i].x *
		             BytesPerPixel);
		PROFILER_ENTER(context->priv->prof_nsc_encode)
		context->encode(context, &data[dataOffset], scanline);
		PROFILER_EXIT(context->priv->prof_nsc_encode)
		PROFILER_ENTER(context->priv->prof_nsc_rle_compress_data)
		nsc_rle_compress_data(context);
		PROFILER_EXIT(context->priv->prof_nsc_rle_compress_data)
		messages[i].LumaPlaneByteCount = context->PlaneByteCount[0];
		messages[i].OrangeChromaPlaneByteCount = context->PlaneByteCount[1];
		messages[i].GreenChromaPlaneByteCount = context->PlaneByteCount[2];
		messages[i].AlphaPlaneByteCount = context->PlaneByteCount[3];
		messages[i].ColorLossLevel = context->ColorLossLevel;
		messages[i].ChromaSubsamplingLevel = context->ChromaSubsamplingLevel;
	}

	context->priv->PlaneBuffers[0] = NULL;
	context->priv->PlaneBuffers[1] = NULL;
	context->priv->PlaneBuffers[2] = NULL;
	context->priv->PlaneBuffers[3] = NULL;
	context->priv->PlaneBuffers[4] = NULL;
	return messages;
fail:

	for (i = 0; i < *numMessages; i++)
		BufferPool_Return(context->priv->PlanePool, messages[i].PlaneBuffer);

	free(messages);
	return NULL;
}

Project: FreeRDP
File: nsc_encode.c

void nsc_message_free(NSC_CONTEXT* context, NSC_MESSAGE* message)
{
	BufferPool_Return(context->priv->PlanePool, message->PlaneBuffer);
}

Project: FreeRDP
File: nsc_encode.c

BOOL nsc_write_message(NSC_CONTEXT* context, wStream* s, NSC_MESSAGE* message)
{
	UINT32 totalPlaneByteCount;
	totalPlaneByteCount = message->LumaPlaneByteCount +
	                      message->OrangeChromaPlaneByteCount +
	                      message->GreenChromaPlaneByteCount + message->AlphaPlaneByteCount;

	if (!Stream_EnsureRemainingCapacity(s, 20 + totalPlaneByteCount))
		return -1;

	Stream_Write_UINT32(s,
	                    message->LumaPlaneByteCount); /* LumaPlaneByteCount (4 bytes) */
	Stream_Write_UINT32(s,
	                    message->OrangeChromaPlaneByteCount); /* OrangeChromaPlaneByteCount (4 bytes) */
	Stream_Write_UINT32(s,
	                    message->GreenChromaPlaneByteCount); /* GreenChromaPlaneByteCount (4 bytes) */
	Stream_Write_UINT32(s,
	                    message->AlphaPlaneByteCount); /* AlphaPlaneByteCount (4 bytes) */
	Stream_Write_UINT8(s, message->ColorLossLevel); /* ColorLossLevel (1 byte) */
	Stream_Write_UINT8(s,
	                   message->ChromaSubsamplingLevel); /* ChromaSubsamplingLevel (1 byte) */
	Stream_Write_UINT16(s, 0); /* Reserved (2 bytes) */

	if (message->LumaPlaneByteCount)
		Stream_Write(s, message->PlaneBuffers[0],
		             message->LumaPlaneByteCount); /* LumaPlane */

	if (message->OrangeChromaPlaneByteCount)
		Stream_Write(s, message->PlaneBuffers[1],
		             message->OrangeChromaPlaneByteCount); /* OrangeChromaPlane */

	if (message->GreenChromaPlaneByteCount)
		Stream_Write(s, message->PlaneBuffers[2],
		             message->GreenChromaPlaneByteCount); /* GreenChromaPlane */

	if (message->AlphaPlaneByteCount)
		Stream_Write(s, message->PlaneBuffers[3],
		             message->AlphaPlaneByteCount); /* AlphaPlane */

	return TRUE;
}

Project: FreeRDP
File: nsc_sse2.c

static void nsc_encode_argb_to_aycocg_sse2(NSC_CONTEXT* context,
        const BYTE* data, UINT32 scanline)
{
	UINT16 x;
	UINT16 y;
	UINT16 rw;
	BYTE ccl;
	const BYTE* src;
	BYTE* yplane = NULL;
	BYTE* coplane = NULL;
	BYTE* cgplane = NULL;
	BYTE* aplane = NULL;
	__m128i r_val;
	__m128i g_val;
	__m128i b_val;
	__m128i a_val;
	__m128i y_val;
	__m128i co_val;
	__m128i cg_val;
	UINT32 tempWidth;
	tempWidth = ROUND_UP_TO(context->width, 8);
	rw = (context->ChromaSubsamplingLevel > 0 ? tempWidth : context->width);
	ccl = context->ColorLossLevel;

	for (y = 0; y < context->height; y++)
	{
		src = data + (context->height - 1 - y) * scanline;
		yplane = context->priv->PlaneBuffers[0] + y * rw;
		coplane = context->priv->PlaneBuffers[1] + y * rw;
		cgplane = context->priv->PlaneBuffers[2] + y * rw;
		aplane = context->priv->PlaneBuffers[3] + y * context->width;

		for (x = 0; x < context->width; x += 8)
		{
			switch (context->format)
			{
				case PIXEL_FORMAT_BGRX32:
					b_val = _mm_set_epi16(*(src + 28), *(src + 24), *(src + 20), *(src + 16),
					                      *(src + 12), *(src + 8), *(src + 4), *src);
					g_val = _mm_set_epi16(*(src + 29), *(src + 25), *(src + 21), *(src + 17),
					                      *(src + 13), *(src + 9), *(src + 5), *(src + 1));
					r_val = _mm_set_epi16(*(src + 30), *(src + 26), *(src + 22), *(src + 18),
					                      *(src + 14), *(src + 10), *(src + 6), *(src + 2));
					a_val = _mm_set1_epi16(0xFF);
					src += 32;
					break;

				case PIXEL_FORMAT_BGRA32:
					b_val = _mm_set_epi16(*(src + 28), *(src + 24), *(src + 20), *(src + 16),
					                      *(src + 12), *(src + 8), *(src + 4), *src);
					g_val = _mm_set_epi16(*(src + 29), *(src + 25), *(src + 21), *(src + 17),
					                      *(src + 13), *(src + 9), *(src + 5), *(src + 1));
					r_val = _mm_set_epi16(*(src + 30), *(src + 26), *(src + 22), *(src + 18),
					                      *(src + 14), *(src + 10), *(src + 6), *(src + 2));
					a_val = _mm_set_epi16(*(src + 31), *(src + 27), *(src + 23), *(src + 19),
					                      *(src + 15), *(src + 11), *(src + 7), *(src + 3));
					src += 32;
					break;

				case PIXEL_FORMAT_RGBX32:
					r_val = _mm_set_epi16(*(src + 28), *(src + 24), *(src + 20), *(src + 16),
					                      *(src + 12), *(src + 8), *(src + 4), *src);
					g_val = _mm_set_epi16(*(src + 29), *(src + 25), *(src + 21), *(src + 17),
					                      *(src + 13), *(src + 9), *(src + 5), *(src + 1));
					b_val = _mm_set_epi16(*(src + 30), *(src + 26), *(src + 22), *(src + 18),
					                      *(src + 14), *(src + 10), *(src + 6), *(src + 2));
					a_val = _mm_set1_epi16(0xFF);
					src += 32;
					break;

				case PIXEL_FORMAT_RGBA32:
					r_val = _mm_set_epi16(*(src + 28), *(src + 24), *(src + 20), *(src + 16),
					                      *(src + 12), *(src + 8), *(src + 4), *src);
					g_val = _mm_set_epi16(*(src + 29), *(src + 25), *(src + 21), *(src + 17),
					                      *(src + 13), *(src + 9), *(src + 5), *(src + 1));
					b_val = _mm_set_epi16(*(src + 30), *(src + 26), *(src + 22), *(src + 18),
					                      *(src + 14), *(src + 10), *(src + 6), *(src + 2));
					a_val = _mm_set_epi16(*(src + 31), *(src + 27), *(src + 23), *(src + 19),
					                      *(src + 15), *(src + 11), *(src + 7), *(src + 3));
					src += 32;
					break;

				case PIXEL_FORMAT_BGR24:
					b_val = _mm_set_epi16(*(src + 21), *(src + 18), *(src + 15), *(src + 12),
					                      *(src + 9), *(src + 6), *(src + 3), *src);
					g_val = _mm_set_epi16(*(src + 22), *(src + 19), *(src + 16), *(src + 13),
					                      *(src + 10), *(src + 7), *(src + 4), *(src + 1));
					r_val = _mm_set_epi16(*(src + 23), *(src + 20), *(src + 17), *(src + 14),
					                      *(src + 11), *(src + 8), *(src + 5), *(src + 2));
					a_val = _mm_set1_epi16(0xFF);
					src += 24;
					break;

				case PIXEL_FORMAT_RGB24:
					r_val = _mm_set_epi16(*(src + 21), *(src + 18), *(src + 15), *(src + 12),
					                      *(src + 9), *(src + 6), *(src + 3), *src);
					g_val = _mm_set_epi16(*(src + 22), *(src + 19), *(src + 16), *(src + 13),
					                      *(src + 10), *(src + 7), *(src + 4), *(src + 1));
					b_val = _mm_set_epi16(*(src + 23), *(src + 20), *(src + 17), *(src + 14),
					                      *(src + 11), *(src + 8), *(src + 5), *(src + 2));
					a_val = _mm_set1_epi16(0xFF);
					src += 24;
					break;

				case PIXEL_FORMAT_BGR16:
					b_val = _mm_set_epi16(
					            (((*(src + 15)) & 0xF8) | ((*(src + 15)) >> 5)),
					            (((*(src + 13)) & 0xF8) | ((*(src + 13)) >> 5)),
					            (((*(src + 11)) & 0xF8) | ((*(src + 11)) >> 5)),
					            (((*(src + 9)) & 0xF8) | ((*(src + 9)) >> 5)),
					            (((*(src + 7)) & 0xF8) | ((*(src + 7)) >> 5)),
					            (((*(src + 5)) & 0xF8) | ((*(src + 5)) >> 5)),
					            (((*(src + 3)) & 0xF8) | ((*(src + 3)) >> 5)),
					            (((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)));
					g_val = _mm_set_epi16(
					            ((((*(src + 15)) & 0x07) << 5) | (((*(src + 14)) & 0xE0) >> 3)),
					            ((((*(src + 13)) & 0x07) << 5) | (((*(src + 12)) & 0xE0) >> 3)),
					            ((((*(src + 11)) & 0x07) << 5) | (((*(src + 10)) & 0xE0) >> 3)),
					            ((((*(src + 9)) & 0x07) << 5) | (((*(src + 8)) & 0xE0) >> 3)),
					            ((((*(src + 7)) & 0x07) << 5) | (((*(src + 6)) & 0xE0) >> 3)),
					            ((((*(src + 5)) & 0x07) << 5) | (((*(src + 4)) & 0xE0) >> 3)),
					            ((((*(src + 3)) & 0x07) << 5) | (((*(src + 2)) & 0xE0) >> 3)),
					            ((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)));
					r_val = _mm_set_epi16(
					            ((((*(src + 14)) & 0x1F) << 3) | (((*(src + 14)) >> 2) & 0x07)),
					            ((((*(src + 12)) & 0x1F) << 3) | (((*(src + 12)) >> 2) & 0x07)),
					            ((((*(src + 10)) & 0x1F) << 3) | (((*(src + 10)) >> 2) & 0x07)),
					            ((((*(src + 8)) & 0x1F) << 3) | (((*(src + 8)) >> 2) & 0x07)),
					            ((((*(src + 6)) & 0x1F) << 3) | (((*(src + 6)) >> 2) & 0x07)),
					            ((((*(src + 4)) & 0x1F) << 3) | (((*(src + 4)) >> 2) & 0x07)),
					            ((((*(src + 2)) & 0x1F) << 3) | (((*(src + 2)) >> 2) & 0x07)),
					            ((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07)));
					a_val = _mm_set1_epi16(0xFF);
					src += 16;
					break;

				case PIXEL_FORMAT_RGB16:
					r_val = _mm_set_epi16(
					            (((*(src + 15)) & 0xF8) | ((*(src + 15)) >> 5)),
					            (((*(src + 13)) & 0xF8) | ((*(src + 13)) >> 5)),
					            (((*(src + 11)) & 0xF8) | ((*(src + 11)) >> 5)),
					            (((*(src + 9)) & 0xF8) | ((*(src + 9)) >> 5)),
					            (((*(src + 7)) & 0xF8) | ((*(src + 7)) >> 5)),
					            (((*(src + 5)) & 0xF8) | ((*(src + 5)) >> 5)),
					            (((*(src + 3)) & 0xF8) | ((*(src + 3)) >> 5)),
					            (((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)));
					g_val = _mm_set_epi16(
					            ((((*(src + 15)) & 0x07) << 5) | (((*(src + 14)) & 0xE0) >> 3)),
					            ((((*(src + 13)) & 0x07) << 5) | (((*(src + 12)) & 0xE0) >> 3)),
					            ((((*(src + 11)) & 0x07) << 5) | (((*(src + 10)) & 0xE0) >> 3)),
					            ((((*(src + 9)) & 0x07) << 5) | (((*(src + 8)) & 0xE0) >> 3)),
					            ((((*(src + 7)) & 0x07) << 5) | (((*(src + 6)) & 0xE0) >> 3)),
					            ((((*(src + 5)) & 0x07) << 5) | (((*(src + 4)) & 0xE0) >> 3)),
					            ((((*(src + 3)) & 0x07) << 5) | (((*(src + 2)) & 0xE0) >> 3)),
					            ((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)));
					b_val = _mm_set_epi16(
					            ((((*(src + 14)) & 0x1F) << 3) | (((*(src + 14)) >> 2) & 0x07)),
					            ((((*(src + 12)) & 0x1F) << 3) | (((*(src + 12)) >> 2) & 0x07)),
					            ((((*(src + 10)) & 0x1F) << 3) | (((*(src + 10)) >> 2) & 0x07)),
					            ((((*(src + 8)) & 0x1F) << 3) | (((*(src + 8)) >> 2) & 0x07)),
					            ((((*(src + 6)) & 0x1F) << 3) | (((*(src + 6)) >> 2) & 0x07)),
					            ((((*(src + 4)) & 0x1F) << 3) | (((*(src + 4)) >> 2) & 0x07)),
					            ((((*(src + 2)) & 0x1F) << 3) | (((*(src + 2)) >> 2) & 0x07)),
					            ((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07)));
					a_val = _mm_set1_epi16(0xFF);
					src += 16;
					break;

				case PIXEL_FORMAT_A4:
					{
						int shift;
						BYTE idx[8];

						for (shift = 7; shift >= 0; shift--)
						{
							idx[shift] = ((*src) >> shift) & 1;
							idx[shift] |= (((*(src + 1)) >> shift) & 1) << 1;
							idx[shift] |= (((*(src + 2)) >> shift) & 1) << 2;
							idx[shift] |= (((*(src + 3)) >> shift) & 1) << 3;
							idx[shift] *= 3;
						}

						r_val = _mm_set_epi16(
						            context->palette[idx[0]],
						            context->palette[idx[1]],
						            context->palette[idx[2]],
						            context->palette[idx[3]],
						            context->palette[idx[4]],
						            context->palette[idx[5]],
						            context->palette[idx[6]],
						            context->palette[idx[7]]);
						g_val = _mm_set_epi16(
						            context->palette[idx[0] + 1],
						            context->palette[idx[1] + 1],
						            context->palette[idx[2] + 1],
						            context->palette[idx[3] + 1],
						            context->palette[idx[4] + 1],
						            context->palette[idx[5] + 1],
						            context->palette[idx[6] + 1],
						            context->palette[idx[7] + 1]);
						b_val = _mm_set_epi16(
						            context->palette[idx[0] + 2],
						            context->palette[idx[1] + 2],
						            context->palette[idx[2] + 2],
						            context->palette[idx[3] + 2],
						            context->palette[idx[4] + 2],
						            context->palette[idx[5] + 2],
						            context->palette[idx[6] + 2],
						            context->palette[idx[7] + 2]);
						src += 4;
					}

					a_val = _mm_set1_epi16(0xFF);
					break;

				case PIXEL_FORMAT_RGB8:
					{
						r_val = _mm_set_epi16(
						            context->palette[(*(src + 7)) * 3],
						            context->palette[(*(src + 6)) * 3],
						            context->palette[(*(src + 5)) * 3],
						            context->palette[(*(src + 4)) * 3],
						            context->palette[(*(src + 3)) * 3],
						            context->palette[(*(src + 2)) * 3],
						            context->palette[(*(src + 1)) * 3],
						            context->palette[(*src) * 3]);
						g_val = _mm_set_epi16(
						            context->palette[(*(src + 7)) * 3 + 1],
						            context->palette[(*(src + 6)) * 3 + 1],
						            context->palette[(*(src + 5)) * 3 + 1],
						            context->palette[(*(src + 4)) * 3 + 1],
						            context->palette[(*(src + 3)) * 3 + 1],
						            context->palette[(*(src + 2)) * 3 + 1],
						            context->palette[(*(src + 1)) * 3 + 1],
						            context->palette[(*src) * 3 + 1]);
						b_val = _mm_set_epi16(
						            context->palette[(*(src + 7)) * 3 + 2],
						            context->palette[(*(src + 6)) * 3 + 2],
						            context->palette[(*(src + 5)) * 3 + 2],
						            context->palette[(*(src + 4)) * 3 + 2],
						            context->palette[(*(src + 3)) * 3 + 2],
						            context->palette[(*(src + 2)) * 3 + 2],
						            context->palette[(*(src + 1)) * 3 + 2],
						            context->palette[(*src) * 3 + 2]);
						src += 8;
					}

					a_val = _mm_set1_epi16(0xFF);
					break;

				default:
					r_val = g_val = b_val = a_val = _mm_set1_epi16(0);
					break;
			}

			y_val = _mm_srai_epi16(r_val, 2);
			y_val = _mm_add_epi16(y_val, _mm_srai_epi16(g_val, 1));
			y_val = _mm_add_epi16(y_val, _mm_srai_epi16(b_val, 2));
			co_val = _mm_sub_epi16(r_val, b_val);
			co_val = _mm_srai_epi16(co_val, ccl);
			cg_val = _mm_sub_epi16(g_val, _mm_srai_epi16(r_val, 1));
			cg_val = _mm_sub_epi16(cg_val, _mm_srai_epi16(b_val, 1));
			cg_val = _mm_srai_epi16(cg_val, ccl);
			y_val = _mm_packus_epi16(y_val, y_val);
			_mm_storeu_si128((__m128i*) yplane, y_val);
			co_val = _mm_packs_epi16(co_val, co_val);
			_mm_storeu_si128((__m128i*) coplane, co_val);
			cg_val = _mm_packs_epi16(cg_val, cg_val);
			_mm_storeu_si128((__m128i*) cgplane, cg_val);
			a_val = _mm_packus_epi16(a_val, a_val);
			_mm_storeu_si128((__m128i*) aplane, a_val);
			yplane += 8;
			coplane += 8;
			cgplane += 8;
			aplane += 8;
		}

		if (context->ChromaSubsamplingLevel > 0 && (context->width % 2) == 1)
		{
			context->priv->PlaneBuffers[0][y * rw + context->width] =
			    context->priv->PlaneBuffers[0][y * rw + context->width - 1];
			context->priv->PlaneBuffers[1][y * rw + context->width] =
			    context->priv->PlaneBuffers[1][y * rw + context->width - 1];
			context->priv->PlaneBuffers[2][y * rw + context->width] =
			    context->priv->PlaneBuffers[2][y * rw + context->width - 1];
		}
	}

	if (context->ChromaSubsamplingLevel > 0 && (y % 2) == 1)
	{
		yplane = context->priv->PlaneBuffers[0] + y * rw;
		coplane = context->priv->PlaneBuffers[1] + y * rw;
		cgplane = context->priv->PlaneBuffers[2] + y * rw;
		CopyMemory(yplane, yplane - rw, rw);
		CopyMemory(coplane, coplane - rw, rw);
		CopyMemory(cgplane, cgplane - rw, rw);
	}
}

Project: FreeRDP
File: nsc_sse2.c

static void nsc_encode_subsampling_sse2(NSC_CONTEXT* context)
{
	UINT16 x;
	UINT16 y;
	BYTE* co_dst;
	BYTE* cg_dst;
	INT8* co_src0;
	INT8* co_src1;
	INT8* cg_src0;
	INT8* cg_src1;
	UINT32 tempWidth;
	UINT32 tempHeight;
	__m128i t;
	__m128i val;
	__m128i mask = _mm_set1_epi16(0xFF);
	tempWidth = ROUND_UP_TO(context->width, 8);
	tempHeight = ROUND_UP_TO(context->height, 2);

	for (y = 0; y < tempHeight >> 1; y++)
	{
		co_dst = context->priv->PlaneBuffers[1] + y * (tempWidth >> 1);
		cg_dst = context->priv->PlaneBuffers[2] + y * (tempWidth >> 1);
		co_src0 = (INT8*) context->priv->PlaneBuffers[1] + (y << 1) * tempWidth;
		co_src1 = co_src0 + tempWidth;
		cg_src0 = (INT8*) context->priv->PlaneBuffers[2] + (y << 1) * tempWidth;
		cg_src1 = cg_src0 + tempWidth;

		for (x = 0; x < tempWidth >> 1; x += 8)
		{
			t = _mm_loadu_si128((__m128i*) co_src0);
			t = _mm_avg_epu8(t, _mm_loadu_si128((__m128i*) co_src1));
			val = _mm_and_si128(_mm_srli_si128(t, 1), mask);
			val = _mm_avg_epu16(val, _mm_and_si128(t, mask));
			val = _mm_packus_epi16(val, val);
			_mm_storeu_si128((__m128i*) co_dst, val);
			co_dst += 8;
			co_src0 += 16;
			co_src1 += 16;
			t = _mm_loadu_si128((__m128i*) cg_src0);
			t = _mm_avg_epu8(t, _mm_loadu_si128((__m128i*) cg_src1));
			val = _mm_and_si128(_mm_srli_si128(t, 1), mask);
			val = _mm_avg_epu16(val, _mm_and_si128(t, mask));
			val = _mm_packus_epi16(val, val);
			_mm_storeu_si128((__m128i*) cg_dst, val);
			cg_dst += 8;
			cg_src0 += 16;
			cg_src1 += 16;
		}
 	}
 }

Project: FreeRDP
File: nsc_sse2.c

 void nsc_init_sse2(NSC_CONTEXT* context)
{
	if (!IsProcessorFeaturePresent(PF_XMMI64_INSTRUCTIONS_AVAILABLE))
		return;

	PROFILER_RENAME(context->priv->prof_nsc_encode, "nsc_encode_sse2");
	context->encode = nsc_encode_sse2;
}