Commit message: libvncserver: fix pointer aliasing/alignment issue

Accessing byte-aligned data through uint16_t pointers can cause crashes
on some platforms or reduce the performance. Therefore ensure a proper
stack alignment.
CVE: CVE-2020-14400
CWE: NVD-CWE-Other
Vulnerability description: An issue was discovered in LibVNCServer before 0.9.13. Byte-aligned data is accessed through uint16_t pointers in libvncserver/translate.c. NOTE: Third parties do not consider this to be a vulnerability as there is no known path of exploitation or cross of a trust boundary

Project: libvncserver
File: translate.c

rfbSetClientColourMap(rfbClientPtr cl, int firstColour, int nColours)
{
    if (cl->screen->serverFormat.trueColour || !cl->readyForSetColourMapEntries) {
	return TRUE;
    }

    if (nColours == 0) {
	nColours = cl->screen->colourMap.count;
    }

    if (cl->format.trueColour) {
	LOCK(cl->updateMutex);
	(*rfbInitColourMapSingleTableFns
	    [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
					     &cl->screen->serverFormat, &cl->format,&cl->screen->colourMap);

	sraRgnDestroy(cl->modifiedRegion);
	cl->modifiedRegion =
	  sraRgnCreateRect(0,0,cl->screen->width,cl->screen->height);
	UNLOCK(cl->updateMutex);

	return TRUE;
    }

    return rfbSendSetColourMapEntries(cl, firstColour, nColours);
}

Project: libvncserver
File: translate.c

rfbTranslateNone(char *table, rfbPixelFormat *in, rfbPixelFormat *out,
                 char *iptr, char *optr, int bytesBetweenInputLines,
                 int width, int height)
{
    int bytesPerOutputLine = width * (out->bitsPerPixel / 8);

    while (height > 0) {
        memcpy(optr, iptr, bytesPerOutputLine);
        iptr += bytesBetweenInputLines;
        optr += bytesPerOutputLine;
        height--;
    }
}

Project: libvncserver
File: translate.c

rfbSetClientColourMapBGR233(rfbClientPtr cl)
{
    union {
        char bytes[sz_rfbSetColourMapEntriesMsg + 256 * 3 * 2];
        rfbSetColourMapEntriesMsg msg;
    } buf;
    rfbSetColourMapEntriesMsg *scme = &buf.msg;
    uint16_t *rgb = (uint16_t *)(&buf.bytes[sz_rfbSetColourMapEntriesMsg]);
    int i, len;
    int r, g, b;

    if (cl->format.bitsPerPixel != 8 ) {
        rfbErr("%s: client not 8 bits per pixel\n",
                "rfbSetClientColourMapBGR233");
        rfbCloseClient(cl);
        return FALSE;
    }
    
    scme->type = rfbSetColourMapEntries;

    scme->firstColour = Swap16IfLE(0);
    scme->nColours = Swap16IfLE(256);

    len = sz_rfbSetColourMapEntriesMsg;

    i = 0;

    for (b = 0; b < 4; b++) {
        for (g = 0; g < 8; g++) {
            for (r = 0; r < 8; r++) {
                rgb[i++] = Swap16IfLE(r * 65535 / 7);
                rgb[i++] = Swap16IfLE(g * 65535 / 7);
                rgb[i++] = Swap16IfLE(b * 65535 / 3);
            }
        }
    }

    len += 256 * 3 * 2;

    if (rfbWriteExact(cl, buf.bytes, len) < 0) {
        rfbLogPerror("rfbSetClientColourMapBGR233: write");
        rfbCloseClient(cl);
        return FALSE;
    }
    return TRUE;
}

Project: libvncserver
File: translate.c

rfbSetClientColourMaps(rfbScreenInfoPtr rfbScreen, int firstColour, int nColours)
{
    rfbClientIteratorPtr i;
    rfbClientPtr cl;

    i = rfbGetClientIterator(rfbScreen);
    while((cl = rfbClientIteratorNext(i)))
      rfbSetClientColourMap(cl, firstColour, nColours);
    rfbReleaseClientIterator(i);
}

Project: libvncserver
File: translate.c

PrintPixelFormat(rfbPixelFormat *pf)
{
    if (pf->bitsPerPixel == 1) {
        rfbLog("  1 bpp, %s sig bit in each byte is leftmost on the screen.\n",
               (pf->bigEndian ? "most" : "least"));
    } else {
        rfbLog("  %d bpp, depth %d%s\n",pf->bitsPerPixel,pf->depth,
               ((pf->bitsPerPixel == 8) ? ""
                : (pf->bigEndian ? ", big endian" : ", little endian")));
        if (pf->trueColour) {
            rfbLog("  true colour: max r %d g %d b %d, shift r %d g %d b %d\n",
                   pf->redMax, pf->greenMax, pf->blueMax,
                   pf->redShift, pf->greenShift, pf->blueShift);
        } else {
            rfbLog("  uses a colour map (not true colour).\n");
        }
    }
}

Project: libvncserver
File: translate.c

rfbSetTranslateFunction(rfbClientPtr cl)
{
    rfbLog("Pixel format for client %s:\n",cl->host);
    PrintPixelFormat(&cl->format);

    /*
     * Check that bits per pixel values are valid
     */

    if ((cl->screen->serverFormat.bitsPerPixel != 8) &&
        (cl->screen->serverFormat.bitsPerPixel != 16) &&
#ifdef LIBVNCSERVER_ALLOW24BPP
	(cl->screen->serverFormat.bitsPerPixel != 24) &&
#endif
        (cl->screen->serverFormat.bitsPerPixel != 32))
    {
        rfbErr("%s: server bits per pixel not 8, 16 or 32 (is %d)\n",
	       "rfbSetTranslateFunction", 
	       cl->screen->serverFormat.bitsPerPixel);
        rfbCloseClient(cl);
        return FALSE;
    }

    if ((cl->format.bitsPerPixel != 8) &&
        (cl->format.bitsPerPixel != 16) &&
#ifdef LIBVNCSERVER_ALLOW24BPP
	(cl->format.bitsPerPixel != 24) &&
#endif
        (cl->format.bitsPerPixel != 32))
    {
        rfbErr("%s: client bits per pixel not 8, 16 or 32\n",
                "rfbSetTranslateFunction");
        rfbCloseClient(cl);
        return FALSE;
    }

    if (!cl->format.trueColour && (cl->format.bitsPerPixel != 8)) {
        rfbErr("rfbSetTranslateFunction: client has colour map "
                "but %d-bit - can only cope with 8-bit colour maps\n",
                cl->format.bitsPerPixel);
        rfbCloseClient(cl);
        return FALSE;
    }

    /*
     * bpp is valid, now work out how to translate
     */

    if (!cl->format.trueColour) {
        /*
         * truecolour -> colour map
         *
         * Set client's colour map to BGR233, then effectively it's
         * truecolour as well
         */

        if (!rfbSetClientColourMapBGR233(cl))
            return FALSE;

        cl->format = BGR233Format;
    }

    /* truecolour -> truecolour */

    if (PF_EQ(cl->format,cl->screen->serverFormat)) {

        /* client & server the same */

        rfbLog("no translation needed\n");
        cl->translateFn = rfbTranslateNone;
        return TRUE;
    }

    if ((cl->screen->serverFormat.bitsPerPixel < 16) ||
        ((!cl->screen->serverFormat.trueColour || !rfbEconomicTranslate) &&
	   (cl->screen->serverFormat.bitsPerPixel == 16))) {

        /* we can use a single lookup table for <= 16 bpp */

        cl->translateFn = rfbTranslateWithSingleTableFns
                              [BPP2OFFSET(cl->screen->serverFormat.bitsPerPixel)]
                                  [BPP2OFFSET(cl->format.bitsPerPixel)];

	if(cl->screen->serverFormat.trueColour)
	  (*rfbInitTrueColourSingleTableFns
	   [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
						   &(cl->screen->serverFormat), &cl->format);
	else
	  (*rfbInitColourMapSingleTableFns
	   [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
						   &(cl->screen->serverFormat), &cl->format,&cl->screen->colourMap);

    } else {

        /* otherwise we use three separate tables for red, green and blue */

        cl->translateFn = rfbTranslateWithRGBTablesFns
                              [BPP2OFFSET(cl->screen->serverFormat.bitsPerPixel)]
                                  [BPP2OFFSET(cl->format.bitsPerPixel)];

        (*rfbInitTrueColourRGBTablesFns
            [BPP2OFFSET(cl->format.bitsPerPixel)]) (&cl->translateLookupTable,
                                             &(cl->screen->serverFormat), &cl->format);
    }

    return TRUE;
}