Commit message: quic: Avoid possible buffer overflow in find_bucket

Proved by fuzzing the code.

Signed-off-by: Frediano Ziglio <freddy77@gmail.com>
Acked-by: Uri Lublin <uril@redhat.com>
CVE: None
Vulnerability description: None

Project: spice-common
File: quic_family_tmpl.c

static void FNAME(golomb_coding)(Encoder *encoder, const BYTE n, const unsigned int l)
{
    encode(encoder, FNAME(golomb_code)(n, l), FNAME(golomb_code_len)(n, l));
}

Project: spice-common
File: quic_family_tmpl.c

static inline unsigned int FNAME(golomb_code_len)(const BYTE n, const unsigned int l)
{
    return VNAME(family).golomb_code_len[n][l];
}

Project: spice-common
File: quic_family_tmpl.c

static inline unsigned int FNAME(golomb_code)(const BYTE n, const unsigned int l)
{
    return VNAME(family).golomb_code[n][l];
}

Project: spice-common
File: quic_family_tmpl.c

static unsigned int FNAME(golomb_decoding)(const unsigned int l, const unsigned int bits,
                                           unsigned int * const codewordlen)
{
    if (bits > VNAME(family).notGRprefixmask[l]) { /*GR*/
        const unsigned int zeroprefix = cnt_l_zeroes(bits);       /* leading zeroes in codeword */
        const unsigned int cwlen = zeroprefix + 1 + l;            /* codeword length */
        (*codewordlen) = cwlen;
        return (zeroprefix << l) | ((bits >> (32 - cwlen)) & bppmask[l]);
    } else { /* not-GR */
        const unsigned int cwlen = VNAME(family).notGRcwlen[l];
        (*codewordlen) = cwlen;
        return VNAME(family).nGRcodewords[l] + ((bits) >> (32 - cwlen) &
                                                bppmask[VNAME(family).notGRsuffixlen[l]]);
    }
}

Project: spice-common
File: quic_family_tmpl.c

static s_bucket *FNAME(find_bucket)(Channel *channel, const unsigned int val)
{
    spice_extra_assert(val < (0x1U << BPC));

    /* The and (&) here is to avoid buffer overflows in case of garbage or malicious
     * attempts. Is much faster then using comparisons and save us from such situations.
     * Note that on normal build the check above won't be compiled as this code path
     * is pretty hot and would cause speed regressions.
     */
    return channel->_buckets_ptrs[val & ((1U << BPC) - 1)];
}

Project: spice-common
File: quic_family_tmpl.c

static void FNAME(update_model)(CommonState *state, s_bucket * const bucket,
                                const BYTE curval)
{
    SPICE_VERIFY(BPC >= 1);
    spice_return_if_fail (bucket != NULL);

    const unsigned int bpp = BPC;
    COUNTER * const pcounters = bucket->pcounters;
    unsigned int i;
    unsigned int bestcode;
    unsigned int bestcodelen;

    /* update counters, find minimum */

    bestcode = bpp - 1;
    bestcodelen = (pcounters[bestcode] += FNAME(golomb_code_len)(curval, bestcode));

    for (i = bpp - 2; i < bpp; i--) { /* NOTE: expression i<bpp for signed int i would be: i>=0 */
        const unsigned int ithcodelen = (pcounters[i] += FNAME(golomb_code_len)(curval, i));

        if (ithcodelen < bestcodelen) {
            bestcode = i;
            bestcodelen = ithcodelen;
        }
    }

    bucket->bestcode = bestcode; /* store the found minimum */

    if (bestcodelen > state->wm_trigger) { /* halving counters? */
        for (i = 0; i < bpp; i++) {
            pcounters[i] >>= 1;
        }
    }
}