Commit message: Xi: integer overflow and unvalidated length in (S)ProcXIBarrierReleasePointer

[jcristau: originally this patch fixed the same issue as commit
 211e05ac85 "Xi: Test exact size of XIBarrierReleasePointer", with the
 addition of these checks]

This addresses CVE-2017-12179

Reviewed-by: Alan Coopersmith <alan.coopersmith@oracle.com>
Reviewed-by: Jeremy Huddleston Sequoia <jeremyhu@apple.com>
Reviewed-by: Julien Cristau <jcristau@debian.org>
Signed-off-by: Jeremy Huddleston Sequoia <jeremyhu@apple.com>
Signed-off-by: Nathan Kidd <nkidd@opentext.com>
Signed-off-by: Julien Cristau <jcristau@debian.org>
CVE: CVE-2017-12179
CWE: CWE-190
Vulnerability description: xorg-x11-server before 1.19.5 was vulnerable to integer overflow in (S)ProcXIBarrierReleasePointer functions allowing malicious X client to cause X server to crash or possibly execute arbitrary code.

Project: xserver
File: xibarriers.c

static struct PointerBarrierDevice *AllocBarrierDevice(void)
{
    struct PointerBarrierDevice *pbd = NULL;

    pbd = malloc(sizeof(struct PointerBarrierDevice));
    if (!pbd)
        return NULL;

    pbd->deviceid = -1; /* must be set by caller */
    pbd->barrier_event_id = 1;
    pbd->release_event_id = 0;
    pbd->hit = FALSE;
    pbd->seen = FALSE;
    xorg_list_init(&pbd->entry);

    return pbd;
}

Project: xserver
File: xibarriers.c

BarrierFreeBarrier(void *data, XID id)
{
    struct PointerBarrierClient *c;
    Time ms = GetTimeInMillis();
    DeviceIntPtr dev = NULL;
    ScreenPtr screen;

    c = container_of(data, struct PointerBarrierClient, barrier);
    screen = c->screen;

    for (dev = inputInfo.devices; dev; dev = dev->next) {
        struct PointerBarrierDevice *pbd;
        int root_x, root_y;
        BarrierEvent ev = {
            .header = ET_Internal,
            .type = ET_BarrierLeave,
            .length = sizeof (BarrierEvent),
            .time = ms,
            /* .deviceid */
            .sourceid = 0,
            .barrierid = c->id,
            .window = c->window,
            .root = screen->root->drawable.id,
            .dx = 0,
            .dy = 0,
            /* .root_x */
            /* .root_y */
            /* .dt */
            /* .event_id */
            .flags = XIBarrierPointerReleased,
        };


        if (dev->type != MASTER_POINTER)
            continue;

        pbd = GetBarrierDevice(c, dev->id);
        if (!pbd->hit)
            continue;

        ev.deviceid = dev->id;
        ev.event_id = pbd->barrier_event_id;
        ev.dt = ms - pbd->last_timestamp;

        GetSpritePosition(dev, &root_x, &root_y);
        ev.root_x = root_x;
        ev.root_y = root_y;

        mieqEnqueue(dev, (InternalEvent *) &ev);
    }

    xorg_list_del(&c->entry);

    FreePointerBarrierClient(c);
    return Success;
}

Project: xserver
File: xibarriers.c

CreatePointerBarrierClient(ClientPtr client,
                           xXFixesCreatePointerBarrierReq * stuff,
                           PointerBarrierClientPtr *client_out)
{
    WindowPtr pWin;
    ScreenPtr screen;
    BarrierScreenPtr cs;
    int err;
    int size;
    int i;
    struct PointerBarrierClient *ret;
    CARD16 *in_devices;
    DeviceIntPtr dev;

    size = sizeof(*ret) + sizeof(DeviceIntPtr) * stuff->num_devices;
    ret = malloc(size);

    if (!ret) {
        return BadAlloc;
    }

    xorg_list_init(&ret->per_device);

    err = dixLookupWindow(&pWin, stuff->window, client, DixReadAccess);
    if (err != Success) {
        client->errorValue = stuff->window;
        goto error;
    }

    screen = pWin->drawable.pScreen;
    cs = GetBarrierScreen(screen);

    ret->screen = screen;
    ret->window = stuff->window;
    ret->num_devices = stuff->num_devices;
    if (ret->num_devices > 0)
        ret->device_ids = (int*)&ret[1];
    else
        ret->device_ids = NULL;

    in_devices = (CARD16 *) &stuff[1];
    for (i = 0; i < stuff->num_devices; i++) {
        int device_id = in_devices[i];
        DeviceIntPtr device;

        if ((err = dixLookupDevice (&device, device_id,
                                    client, DixReadAccess))) {
            client->errorValue = device_id;
            goto error;
        }

        if (!IsMaster (device)) {
            client->errorValue = device_id;
            err = BadDevice;
            goto error;
        }

        ret->device_ids[i] = device_id;
    }

    /* Alloc one per master pointer, they're the ones that can be blocked */
    xorg_list_init(&ret->per_device);
    nt_list_for_each_entry(dev, inputInfo.devices, next) {
        struct PointerBarrierDevice *pbd;

        if (dev->type != MASTER_POINTER)
            continue;

        pbd = AllocBarrierDevice();
        if (!pbd) {
            err = BadAlloc;
            goto error;
        }
        pbd->deviceid = dev->id;

        xorg_list_add(&pbd->entry, &ret->per_device);
    }

    ret->id = stuff->barrier;
    ret->barrier.x1 = stuff->x1;
    ret->barrier.x2 = stuff->x2;
    ret->barrier.y1 = stuff->y1;
    ret->barrier.y2 = stuff->y2;
    sort_min_max(&ret->barrier.x1, &ret->barrier.x2);
    sort_min_max(&ret->barrier.y1, &ret->barrier.y2);
    ret->barrier.directions = stuff->directions & 0x0f;
    if (barrier_is_horizontal(&ret->barrier))
        ret->barrier.directions &= ~(BarrierPositiveX | BarrierNegativeX);
    if (barrier_is_vertical(&ret->barrier))
        ret->barrier.directions &= ~(BarrierPositiveY | BarrierNegativeY);
    xorg_list_add(&ret->entry, &cs->barriers);

    *client_out = ret;
    return Success;

 error:
    *client_out = NULL;
    FreePointerBarrierClient(ret);
    return err;
}

Project: xserver
File: xibarriers.c

static struct PointerBarrierDevice *GetBarrierDevice(struct PointerBarrierClient *c, int deviceid)
{
    struct PointerBarrierDevice *pbd = NULL;

    xorg_list_for_each_entry(pbd, &c->per_device, entry) {
        if (pbd->deviceid == deviceid)
            break;
    }

    BUG_WARN(!pbd);
    return pbd;
}

Project: xserver
File: xibarriers.c

void XIBarrierNewMasterDevice(ClientPtr client, int deviceid)
{
    FindClientResourcesByType(client, PointerBarrierType, add_master_func, &deviceid);
}

Project: xserver
File: xibarriers.c

void XIBarrierRemoveMasterDevice(ClientPtr client, int deviceid)
{
    FindClientResourcesByType(client, PointerBarrierType, remove_master_func, &deviceid);
}

Project: xserver
File: xibarriers.c

XICreatePointerBarrier(ClientPtr client,
                       xXFixesCreatePointerBarrierReq * stuff)
{
    int err;
    struct PointerBarrierClient *barrier;
    struct PointerBarrier b;

    b.x1 = stuff->x1;
    b.x2 = stuff->x2;
    b.y1 = stuff->y1;
    b.y2 = stuff->y2;

    if (!barrier_is_horizontal(&b) && !barrier_is_vertical(&b))
        return BadValue;

    /* no 0-sized barriers */
    if (barrier_is_horizontal(&b) && barrier_is_vertical(&b))
        return BadValue;

    /* no infinite barriers on the wrong axis */
    if (barrier_is_horizontal(&b) && (b.y1 < 0 || b.y2 < 0))
        return BadValue;

    if (barrier_is_vertical(&b) && (b.x1 < 0 || b.x2 < 0))
        return BadValue;

    if ((err = CreatePointerBarrierClient(client, stuff, &barrier)))
        return err;

    if (!AddResource(stuff->barrier, PointerBarrierType, &barrier->barrier))
        return BadAlloc;

    return Success;
}

Project: xserver
File: xibarriers.c

XIDestroyPointerBarrier(ClientPtr client,
                        xXFixesDestroyPointerBarrierReq * stuff)
{
    int err;
    void *barrier;

    err = dixLookupResourceByType((void **) &barrier, stuff->barrier,
                                  PointerBarrierType, client, DixDestroyAccess);
    if (err != Success) {
        client->errorValue = stuff->barrier;
        return err;
    }

    if (CLIENT_ID(stuff->barrier) != client->index)
        return BadAccess;

    FreeResource(stuff->barrier, RT_NONE);
    return Success;
}

Project: xserver
File: xibarriers.c

barrier_blocks_device(struct PointerBarrierClient *client,
                      DeviceIntPtr dev)
{
    int i;
    int master_id;

    /* Clients with no devices are treated as
     * if they specified XIAllDevices. */
    if (client->num_devices == 0)
        return TRUE;

    master_id = GetMaster(dev, POINTER_OR_FLOAT)->id;

    for (i = 0; i < client->num_devices; i++) {
        int device_id = client->device_ids[i];
        if (device_id == XIAllDevices ||
            device_id == XIAllMasterDevices ||
            device_id == master_id)
            return TRUE;
    }

    return FALSE;
}

Project: xserver
File: xibarriers.c

barrier_clamp_to_barrier(struct PointerBarrier *barrier, int dir, int *x,
                         int *y)
{
    if (barrier_is_vertical(barrier)) {
        if ((dir & BarrierNegativeX) & ~barrier->directions)
            *x = barrier->x1;
        if ((dir & BarrierPositiveX) & ~barrier->directions)
            *x = barrier->x1 - 1;
    }
    if (barrier_is_horizontal(barrier)) {
        if ((dir & BarrierNegativeY) & ~barrier->directions)
            *y = barrier->y1;
        if ((dir & BarrierPositiveY) & ~barrier->directions)
            *y = barrier->y1 - 1;
    }
}

Project: xserver
File: xibarriers.c

barrier_find_nearest(BarrierScreenPtr cs, DeviceIntPtr dev,
                     int dir,
                     int x1, int y1, int x2, int y2)
{
    struct PointerBarrierClient *c, *nearest = NULL;
    double min_distance = INT_MAX;      /* can't get higher than that in X anyway */

    xorg_list_for_each_entry(c, &cs->barriers, entry) {
        struct PointerBarrier *b = &c->barrier;
        struct PointerBarrierDevice *pbd;
        double distance;

        pbd = GetBarrierDevice(c, dev->id);
        if (pbd->seen)
            continue;

        if (!barrier_is_blocking_direction(b, dir))
            continue;

        if (!barrier_blocks_device(c, dev))
            continue;

        if (barrier_is_blocking(b, x1, y1, x2, y2, &distance)) {
            if (min_distance > distance) {
                min_distance = distance;
                nearest = c;
            }
        }
    }

    return nearest;
}

Project: xserver
File: xibarriers.c

barrier_get_direction(int x1, int y1, int x2, int y2)
{
    int direction = 0;

    /* which way are we trying to go */
    if (x2 > x1)
        direction |= BarrierPositiveX;
    if (x2 < x1)
        direction |= BarrierNegativeX;
    if (y2 > y1)
        direction |= BarrierPositiveY;
    if (y2 < y1)
        direction |= BarrierNegativeY;

    return direction;
}

Project: xserver
File: xibarriers.c

barrier_inside_hit_box(struct PointerBarrier *barrier, int x, int y)
{
    int x1, x2, y1, y2;
    int dir;

    x1 = barrier->x1;
    x2 = barrier->x2;
    y1 = barrier->y1;
    y2 = barrier->y2;
    dir = ~(barrier->directions);

    if (barrier_is_vertical(barrier)) {
        if (dir & BarrierPositiveX)
            x1 -= HIT_EDGE_EXTENTS;
        if (dir & BarrierNegativeX)
            x2 += HIT_EDGE_EXTENTS;
    }
    if (barrier_is_horizontal(barrier)) {
        if (dir & BarrierPositiveY)
            y1 -= HIT_EDGE_EXTENTS;
        if (dir & BarrierNegativeY)
            y2 += HIT_EDGE_EXTENTS;
    }

    return x >= x1 && x <= x2 && y >= y1 && y <= y2;
}

Project: xserver
File: xibarriers.c

barrier_is_blocking(const struct PointerBarrier * barrier,
                    int x1, int y1, int x2, int y2, double *distance)
{
    if (barrier_is_vertical(barrier)) {
        float t, y;
        t = T(barrier->x1, x1, x2);
        if (t < 0 || t > 1)
            return FALSE;

        /* Edge case: moving away from barrier. */
        if (x2 > x1 && t == 0)
            return FALSE;

        y = F(t, y1, y2);
        if (!inside_segment(y, barrier->y1, barrier->y2))
            return FALSE;

        *distance = sqrt((pow(y - y1, 2) + pow(barrier->x1 - x1, 2)));
        return TRUE;
    }
    else {
        float t, x;
        t = T(barrier->y1, y1, y2);
        if (t < 0 || t > 1)
            return FALSE;

        /* Edge case: moving away from barrier. */
        if (y2 > y1 && t == 0)
            return FALSE;

        x = F(t, x1, x2);
        if (!inside_segment(x, barrier->x1, barrier->x2))
            return FALSE;

        *distance = sqrt((pow(x - x1, 2) + pow(barrier->y1 - y1, 2)));
        return TRUE;
    }
}

Project: xserver
File: xibarriers.c

barrier_is_blocking_direction(const struct PointerBarrier * barrier,
                              int direction)
{
    /* Barriers define which way is ok, not which way is blocking */
    return (barrier->directions & direction) != direction;
}

Project: xserver
File: xibarriers.c

barrier_is_horizontal(const struct PointerBarrier *barrier)
{
    return barrier->y1 == barrier->y2;
}

Project: xserver
File: xibarriers.c

input_constrain_cursor(DeviceIntPtr dev, ScreenPtr screen,
                       int current_x, int current_y,
                       int dest_x, int dest_y,
                       int *out_x, int *out_y,
                       int *nevents, InternalEvent* events)
{
    /* Clamped coordinates here refer to screen edge clamping. */
    BarrierScreenPtr cs = GetBarrierScreen(screen);
    int x = dest_x,
        y = dest_y;
    int dir;
    struct PointerBarrier *nearest = NULL;
    PointerBarrierClientPtr c;
    Time ms = GetTimeInMillis();
    BarrierEvent ev = {
        .header = ET_Internal,
        .type = 0,
        .length = sizeof (BarrierEvent),
        .time = ms,
        .deviceid = dev->id,
        .sourceid = dev->id,
        .dx = dest_x - current_x,
        .dy = dest_y - current_y,
        .root = screen->root->drawable.id,
    };
    InternalEvent *barrier_events = events;
    DeviceIntPtr master;

    if (nevents)
        *nevents = 0;

    if (xorg_list_is_empty(&cs->barriers) || IsFloating(dev))
        goto out;

    /**
     * This function is only called for slave devices, but pointer-barriers
     * are for master-devices only. Flip the device to the master here,
     * continue with that.
     */
    master = GetMaster(dev, MASTER_POINTER);

    /* How this works:
     * Given the origin and the movement vector, get the nearest barrier
     * to the origin that is blocking the movement.
     * Clamp to that barrier.
     * Then, check from the clamped intersection to the original
     * destination, again finding the nearest barrier and clamping.
     */
    dir = barrier_get_direction(current_x, current_y, x, y);

    while (dir != 0) {
        int new_sequence;
        struct PointerBarrierDevice *pbd;

        c = barrier_find_nearest(cs, master, dir, current_x, current_y, x, y);
        if (!c)
            break;

        nearest = &c->barrier;

        pbd = GetBarrierDevice(c, master->id);
        new_sequence = !pbd->hit;

        pbd->seen = TRUE;
        pbd->hit = TRUE;

        if (pbd->barrier_event_id == pbd->release_event_id)
            continue;

        ev.type = ET_BarrierHit;
        barrier_clamp_to_barrier(nearest, dir, &x, &y);

        if (barrier_is_vertical(nearest)) {
            dir &= ~(BarrierNegativeX | BarrierPositiveX);
            current_x = x;
        }
        else if (barrier_is_horizontal(nearest)) {
            dir &= ~(BarrierNegativeY | BarrierPositiveY);
            current_y = y;
        }

        ev.flags = 0;
        ev.event_id = pbd->barrier_event_id;
        ev.barrierid = c->id;

        ev.dt = new_sequence ? 0 : ms - pbd->last_timestamp;
        ev.window = c->window;
        pbd->last_timestamp = ms;

        /* root x/y is filled in later */

        barrier_events->barrier_event = ev;
        barrier_events++;
        *nevents += 1;
    }

    xorg_list_for_each_entry(c, &cs->barriers, entry) {
        struct PointerBarrierDevice *pbd;
        int flags = 0;

        pbd = GetBarrierDevice(c, master->id);
        pbd->seen = FALSE;
        if (!pbd->hit)
            continue;

        if (barrier_inside_hit_box(&c->barrier, x, y))
            continue;

        pbd->hit = FALSE;

        ev.type = ET_BarrierLeave;

        if (pbd->barrier_event_id == pbd->release_event_id)
            flags |= XIBarrierPointerReleased;

        ev.flags = flags;
        ev.event_id = pbd->barrier_event_id;
        ev.barrierid = c->id;

        ev.dt = ms - pbd->last_timestamp;
        ev.window = c->window;
        pbd->last_timestamp = ms;

        /* root x/y is filled in later */

        barrier_events->barrier_event = ev;
        barrier_events++;
        *nevents += 1;

        /* If we've left the hit box, this is the
         * start of a new event ID. */
        pbd->barrier_event_id++;
    }

 out:
    *out_x = x;
    *out_y = y;
}

Project: xserver
File: xibarriers.c

inside_segment(int v, int v1, int v2)
{
    if (v1 < 0 && v2 < 0) /* line */
        return TRUE;
    else if (v1 < 0)      /* ray */
        return v <= v2;
    else if (v2 < 0)      /* ray */
        return v >= v1;
    else                  /* line segment */
        return v >= v1 && v <= v2;
}

Project: xserver
File: xibarriers.c

static void remove_master_func(void *res, XID id, void *devid)
{
    struct PointerBarrierDevice *pbd;
    struct PointerBarrierClient *barrier;
    struct PointerBarrier *b;
    DeviceIntPtr dev;
    int *deviceid = devid;
    int rc;
    Time ms = GetTimeInMillis();

    rc = dixLookupDevice(&dev, *deviceid, serverClient, DixSendAccess);
    if (rc != Success)
        return;

    b = res;
    barrier = container_of(b, struct PointerBarrierClient, barrier);

    pbd = GetBarrierDevice(barrier, *deviceid);

    if (pbd->hit) {
        BarrierEvent ev = {
            .header = ET_Internal,
            .type =ET_BarrierLeave,
            .length = sizeof (BarrierEvent),
            .time = ms,
            .deviceid = *deviceid,
            .sourceid = 0,
            .dx = 0,
            .dy = 0,
            .root = barrier->screen->root->drawable.id,
            .window = barrier->window,
            .dt = ms - pbd->last_timestamp,
            .flags = XIBarrierPointerReleased,
            .event_id = pbd->barrier_event_id,
            .barrierid = barrier->id,
        };

        mieqEnqueue(dev, (InternalEvent *) &ev);
    }

    xorg_list_del(&pbd->entry);
    free(pbd);
}

Project: xserver
File: xibarriers.c

sort_min_max(INT16 *a, INT16 *b)
{
    INT16 A, B;
    if (*a < 0 || *b < 0)
        return;
    A = *a;
    B = *b;
    *a = min(A, B);
    *b = max(A, B);
}