Commit message: dbe: Unvalidated variable-length request in ProcDbeGetVisualInfo (CVE-2017-12177)

v2: Protect against integer overflow (Alan Coopersmith)

Reviewed-by: Alan Coopersmith <alan.coopersmith@oracle.com>
Reviewed-by: Jeremy Huddleston Sequoia <jeremyhu@apple.com>
Reviewed-by: Julien Cristau <jcristau@debian.org>
Signed-off-by: Nathan Kidd <nkidd@opentext.com>
Signed-off-by: Julien Cristau <jcristau@debian.org>
CVE: CVE-2017-12177
CWE: CWE-190
Vulnerability description: xorg-x11-server before 1.19.5 was vulnerable to integer overflow in ProcDbeGetVisualInfo function allowing malicious X client to cause X server to crash or possibly execute arbitrary code.

Project: xserver
File: dbe.c

DbeStubScreen(DbeScreenPrivPtr pDbeScreenPriv, int *nStubbedScreens)
{
    /* Stub DIX. */
    pDbeScreenPriv->SetupBackgroundPainter = NULL;

    /* Do not unwrap PositionWindow nor DestroyWindow.  If the DDX
     * initialization function failed, we assume that it did not wrap
     * PositionWindow.  Also, DestroyWindow is only wrapped if the DDX
     * initialization function succeeded.
     */

    /* Stub DDX. */
    pDbeScreenPriv->GetVisualInfo = NULL;
    pDbeScreenPriv->AllocBackBufferName = NULL;
    pDbeScreenPriv->SwapBuffers = NULL;
    pDbeScreenPriv->WinPrivDelete = NULL;

    (*nStubbedScreens)++;

}                               /* DbeStubScreen() */

Project: xserver
File: dbe.c

ProcDbeAllocateBackBufferName(ClientPtr client)
{
    REQUEST(xDbeAllocateBackBufferNameReq);
    WindowPtr pWin;
    DbeScreenPrivPtr pDbeScreenPriv;
    DbeWindowPrivPtr pDbeWindowPriv;
    XdbeScreenVisualInfo scrVisInfo;
    register int i;
    Bool visualMatched = FALSE;
    xDbeSwapAction swapAction;
    VisualID visual;
    int status;
    int add_index;

    REQUEST_SIZE_MATCH(xDbeAllocateBackBufferNameReq);

    /* The window must be valid. */
    status = dixLookupWindow(&pWin, stuff->window, client, DixManageAccess);
    if (status != Success)
        return status;

    /* The window must be InputOutput. */
    if (pWin->drawable.class != InputOutput) {
        return BadMatch;
    }

    /* The swap action must be valid. */
    swapAction = stuff->swapAction;     /* use local var for performance. */
    if ((swapAction != XdbeUndefined) &&
        (swapAction != XdbeBackground) &&
        (swapAction != XdbeUntouched) && (swapAction != XdbeCopied)) {
        return BadValue;
    }

    /* The id must be in range and not already in use. */
    LEGAL_NEW_RESOURCE(stuff->buffer, client);

    /* The visual of the window must be in the list returned by
     * GetVisualInfo.
     */
    pDbeScreenPriv = DBE_SCREEN_PRIV_FROM_WINDOW(pWin);
    if (!pDbeScreenPriv->GetVisualInfo)
        return BadMatch;        /* screen doesn't support double buffering */

    if (!(*pDbeScreenPriv->GetVisualInfo) (pWin->drawable.pScreen, &scrVisInfo)) {
        /* GetVisualInfo() failed to allocate visual info data. */
        return BadAlloc;
    }

    /* See if the window's visual is on the list. */
    visual = wVisual(pWin);
    for (i = 0; (i < scrVisInfo.count) && !visualMatched; i++) {
        if (scrVisInfo.visinfo[i].visual == visual) {
            visualMatched = TRUE;
        }
    }

    /* Free what was allocated by the GetVisualInfo() call above. */
    free(scrVisInfo.visinfo);

    if (!visualMatched) {
        return BadMatch;
    }

    if ((pDbeWindowPriv = DBE_WINDOW_PRIV(pWin)) == NULL) {
        /* There is no buffer associated with the window.
         * Allocate a window priv.
         */

        pDbeWindowPriv = calloc(1, sizeof(DbeWindowPrivRec));
        if (!pDbeWindowPriv)
            return BadAlloc;

        /* Fill out window priv information. */
        pDbeWindowPriv->pWindow = pWin;
        pDbeWindowPriv->width = pWin->drawable.width;
        pDbeWindowPriv->height = pWin->drawable.height;
        pDbeWindowPriv->x = pWin->drawable.x;
        pDbeWindowPriv->y = pWin->drawable.y;
        pDbeWindowPriv->nBufferIDs = 0;

        /* Set the buffer ID array pointer to the initial (static) array). */
        pDbeWindowPriv->IDs = pDbeWindowPriv->initIDs;

        /* Initialize the buffer ID list. */
        pDbeWindowPriv->maxAvailableIDs = DBE_INIT_MAX_IDS;
        pDbeWindowPriv->IDs[0] = stuff->buffer;

        add_index = 0;
        for (i = 0; i < DBE_INIT_MAX_IDS; i++) {
            pDbeWindowPriv->IDs[i] = DBE_FREE_ID_ELEMENT;
        }

        /* Actually connect the window priv to the window. */
        dixSetPrivate(&pWin->devPrivates, dbeWindowPrivKey, pDbeWindowPriv);

    }                           /* if -- There is no buffer associated with the window. */

    else {
        /* A buffer is already associated with the window.
         * Add the new buffer ID to the array, reallocating the array memory
         * if necessary.
         */

        /* Determine if there is a free element in the ID array. */
        for (i = 0; i < pDbeWindowPriv->maxAvailableIDs; i++) {
            if (pDbeWindowPriv->IDs[i] == DBE_FREE_ID_ELEMENT) {
                /* There is still room in the ID array. */
                break;
            }
        }

        if (i == pDbeWindowPriv->maxAvailableIDs) {
            /* No more room in the ID array -- reallocate another array. */
            XID *pIDs;

            /* Setup an array pointer for the realloc operation below. */
            if (pDbeWindowPriv->maxAvailableIDs == DBE_INIT_MAX_IDS) {
                /* We will malloc a new array. */
                pIDs = NULL;
            }
            else {
                /* We will realloc a new array. */
                pIDs = pDbeWindowPriv->IDs;
            }

            /* malloc/realloc a new array and initialize all elements to 0. */
            pDbeWindowPriv->IDs =
                reallocarray(pIDs,
                             pDbeWindowPriv->maxAvailableIDs + DBE_INCR_MAX_IDS,
                             sizeof(XID));
            if (!pDbeWindowPriv->IDs) {
                return BadAlloc;
            }
            memset(&pDbeWindowPriv->IDs[pDbeWindowPriv->nBufferIDs], 0,
                   (pDbeWindowPriv->maxAvailableIDs + DBE_INCR_MAX_IDS -
                    pDbeWindowPriv->nBufferIDs) * sizeof(XID));

            if (pDbeWindowPriv->maxAvailableIDs == DBE_INIT_MAX_IDS) {
                /* We just went from using the initial (static) array to a
                 * newly allocated array.  Copy the IDs from the initial array
                 * to the new array.
                 */
                memcpy(pDbeWindowPriv->IDs, pDbeWindowPriv->initIDs,
                       DBE_INIT_MAX_IDS * sizeof(XID));
            }

            pDbeWindowPriv->maxAvailableIDs += DBE_INCR_MAX_IDS;
        }

        add_index = i;

    }                           /* else -- A buffer is already associated with the window. */

    /* Call the DDX routine to allocate the back buffer. */
    status = (*pDbeScreenPriv->AllocBackBufferName) (pWin, stuff->buffer,
                                                     stuff->swapAction);

    if (status == Success) {
        pDbeWindowPriv->IDs[add_index] = stuff->buffer;
        if (!AddResource(stuff->buffer, dbeWindowPrivResType,
                         (void *) pDbeWindowPriv)) {
            pDbeWindowPriv->IDs[add_index] = DBE_FREE_ID_ELEMENT;

            if (pDbeWindowPriv->nBufferIDs == 0) {
                status = BadAlloc;
                goto out_free;
            }
        }
    }
    else {
        /* The DDX buffer allocation routine failed for the first buffer of
         * this window.
         */
        if (pDbeWindowPriv->nBufferIDs == 0) {
            goto out_free;
        }
    }

    /* Increment the number of buffers (XIDs) associated with this window. */
    pDbeWindowPriv->nBufferIDs++;

    /* Set swap action on all calls. */
    pDbeWindowPriv->swapAction = stuff->swapAction;

    return status;

 out_free:
    dixSetPrivate(&pWin->devPrivates, dbeWindowPrivKey, NULL);
    free(pDbeWindowPriv);
    return status;

}                               /* ProcDbeAllocateBackBufferName() */

Project: xserver
File: dbe.c

ProcDbeDeallocateBackBufferName(ClientPtr client)
{
    REQUEST(xDbeDeallocateBackBufferNameReq);
    DbeWindowPrivPtr pDbeWindowPriv;
    int rc, i;
    void *val;

    REQUEST_SIZE_MATCH(xDbeDeallocateBackBufferNameReq);

    /* Buffer name must be valid */
    rc = dixLookupResourceByType((void **) &pDbeWindowPriv, stuff->buffer,
                                 dbeWindowPrivResType, client,
                                 DixDestroyAccess);
    if (rc != Success)
        return rc;

    rc = dixLookupResourceByType(&val, stuff->buffer, dbeDrawableResType,
                                 client, DixDestroyAccess);
    if (rc != Success)
        return rc;

    /* Make sure that the id is valid for the window.
     * This is paranoid code since we already looked up the ID by type
     * above.
     */

    for (i = 0; i < pDbeWindowPriv->nBufferIDs; i++) {
        /* Loop through the ID list to find the ID. */
        if (pDbeWindowPriv->IDs[i] == stuff->buffer) {
            break;
        }
    }

    if (i == pDbeWindowPriv->nBufferIDs) {
        /* We did not find the ID in the ID list. */
        client->errorValue = stuff->buffer;
        return dbeErrorBase + DbeBadBuffer;
    }

    FreeResource(stuff->buffer, RT_NONE);

    return Success;

}                               /* ProcDbeDeallocateBackBufferName() */

Project: xserver
File: dbe.c

ProcDbeGetVersion(ClientPtr client)
{
    /* REQUEST(xDbeGetVersionReq); */
    xDbeGetVersionReply rep = {
        .type = X_Reply,
        .sequenceNumber = client->sequence,
        .length = 0,
        .majorVersion = DBE_MAJOR_VERSION,
        .minorVersion = DBE_MINOR_VERSION
    };

    REQUEST_SIZE_MATCH(xDbeGetVersionReq);

    if (client->swapped) {
        swaps(&rep.sequenceNumber);
    }

    WriteToClient(client, sizeof(xDbeGetVersionReply), &rep);

    return Success;

}                               /* ProcDbeGetVersion() */

Project: xserver
File: dbe.c

ProcDbeSwapBuffers(ClientPtr client)
{
    REQUEST(xDbeSwapBuffersReq);
    WindowPtr pWin;
    DbeScreenPrivPtr pDbeScreenPriv;
    DbeSwapInfoPtr swapInfo;
    xDbeSwapInfo *dbeSwapInfo;
    int error;
    unsigned int i, j;
    unsigned int nStuff;
    int nStuff_i;       /* DDX API requires int for nStuff */

    REQUEST_AT_LEAST_SIZE(xDbeSwapBuffersReq);
    nStuff = stuff->n;          /* use local variable for performance. */

    if (nStuff == 0) {
        REQUEST_SIZE_MATCH(xDbeSwapBuffersReq);
        return Success;
    }

    if (nStuff > UINT32_MAX / sizeof(DbeSwapInfoRec))
        return BadAlloc;
    REQUEST_FIXED_SIZE(xDbeSwapBuffersReq, nStuff * sizeof(xDbeSwapInfo));

    /* Get to the swap info appended to the end of the request. */
    dbeSwapInfo = (xDbeSwapInfo *) &stuff[1];

    /* Allocate array to record swap information. */
    swapInfo = xallocarray(nStuff, sizeof(DbeSwapInfoRec));
    if (swapInfo == NULL) {
        return BadAlloc;
    }

    for (i = 0; i < nStuff; i++) {
        /* Check all windows to swap. */

        /* Each window must be a valid window - BadWindow. */
        error = dixLookupWindow(&pWin, dbeSwapInfo[i].window, client,
                                DixWriteAccess);
        if (error != Success) {
            free(swapInfo);
            return error;
        }

        /* Each window must be double-buffered - BadMatch. */
        if (DBE_WINDOW_PRIV(pWin) == NULL) {
            free(swapInfo);
            return BadMatch;
        }

        /* Each window must only be specified once - BadMatch. */
        for (j = i + 1; j < nStuff; j++) {
            if (dbeSwapInfo[i].window == dbeSwapInfo[j].window) {
                free(swapInfo);
                return BadMatch;
            }
        }

        /* Each swap action must be valid - BadValue. */
        if ((dbeSwapInfo[i].swapAction != XdbeUndefined) &&
            (dbeSwapInfo[i].swapAction != XdbeBackground) &&
            (dbeSwapInfo[i].swapAction != XdbeUntouched) &&
            (dbeSwapInfo[i].swapAction != XdbeCopied)) {
            free(swapInfo);
            return BadValue;
        }

        /* Everything checks out OK.  Fill in the swap info array. */
        swapInfo[i].pWindow = pWin;
        swapInfo[i].swapAction = dbeSwapInfo[i].swapAction;

    }                           /* for (i = 0; i < nStuff; i++) */

    /* Call the DDX routine to perform the swap(s).  The DDX routine should
     * scan the swap list (swap info), swap any buffers that it knows how to
     * handle, delete them from the list, and update nStuff to indicate how
     * many windows it did not handle.
     *
     * This scheme allows a range of sophistication in the DDX SwapBuffers()
     * implementation.  Naive implementations could just swap the first buffer
     * in the list, move the last buffer to the front, decrement nStuff, and
     * return.  The next level of sophistication could be to scan the whole
     * list for windows on the same screen.  Up another level, the DDX routine
     * could deal with cross-screen synchronization.
     */

    nStuff_i = nStuff;
    while (nStuff_i > 0) {
        pDbeScreenPriv = DBE_SCREEN_PRIV_FROM_WINDOW(swapInfo[0].pWindow);
        error = (*pDbeScreenPriv->SwapBuffers) (client, &nStuff_i, swapInfo);
        if (error != Success) {
            free(swapInfo);
            return error;
        }
    }

    free(swapInfo);
    return Success;

}                               /* ProcDbeSwapBuffers() */