Commit message: avcodec/scpr: Fix multiple runtime error: index 256 out of bounds for type 'unsigned int [256]'

Fixes: 1519/clusterfuzz-testcase-minimized-5286680976162816

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
CVE: CVE-2017-9995
CWE: CWE-119
Vulnerability description: libavcodec/scpr.c in FFmpeg 3.3 before 3.3.1 does not properly validate height and width data, which allows remote attackers to cause a denial of service (heap-based buffer overflow and application crash) or possibly have unspecified other impact via a crafted file.

Project: FFmpeg
File: scpr.c

static int decompress_i(AVCodecContext *avctx, uint32_t *dst, int linesize)
{
    SCPRContext *s = avctx->priv_data;
    GetByteContext *gb = &s->gb;
    int cx = 0, cx1 = 0, k = 0, clr = 0;
    int run, r, g, b, off, y = 0, x = 0, z, ret;
    unsigned backstep = linesize - avctx->width;
    const int cxshift = s->cxshift;
    unsigned lx, ly, ptype;

    reinit_tables(s);
    bytestream2_skip(gb, 2);
    init_rangecoder(&s->rc, gb);

    while (k < avctx->width + 1) {
        ret = decode_unit(s, &s->pixel_model[0][cx + cx1], 400, &r);
        if (ret < 0)
            return ret;

        cx1 = (cx << 6) & 0xFC0;
        cx = r >> cxshift;
        ret = decode_unit(s, &s->pixel_model[1][cx + cx1], 400, &g);
        if (ret < 0)
            return ret;

        cx1 = (cx << 6) & 0xFC0;
        cx = g >> cxshift;
        ret = decode_unit(s, &s->pixel_model[2][cx + cx1], 400, &b);
        if (ret < 0)
            return ret;

        cx1 = (cx << 6) & 0xFC0;
        cx = b >> cxshift;

        ret = decode_value(s, s->run_model[0], 256, 400, &run);
        if (ret < 0)
            return ret;

        clr = (b << 16) + (g << 8) + r;
        k += run;
        while (run-- > 0) {
            if (y >= avctx->height)
                return AVERROR_INVALIDDATA;

            dst[y * linesize + x] = clr;
            lx = x;
            ly = y;
            x++;
            if (x >= avctx->width) {
                x = 0;
                y++;
            }
        }
    }
    off = -linesize - 1;
    ptype = 0;

    while (x < avctx->width && y < avctx->height) {
        ret = decode_value(s, s->op_model[ptype], 6, 1000, &ptype);
        if (ret < 0)
            return ret;
        if (ptype == 0) {
            ret = decode_unit(s, &s->pixel_model[0][cx + cx1], 400, &r);
            if (ret < 0)
                return ret;

            cx1 = (cx << 6) & 0xFC0;
            cx = r >> cxshift;
            ret = decode_unit(s, &s->pixel_model[1][cx + cx1], 400, &g);
            if (ret < 0)
                return ret;

            cx1 = (cx << 6) & 0xFC0;
            cx = g >> cxshift;
            ret = decode_unit(s, &s->pixel_model[2][cx + cx1], 400, &b);
            if (ret < 0)
                return ret;

            clr = (b << 16) + (g << 8) + r;
        }
        if (ptype > 5)
            return AVERROR_INVALIDDATA;
        ret = decode_value(s, s->run_model[ptype], 256, 400, &run);
        if (ret < 0)
            return ret;

        switch (ptype) {
        case 0:
            while (run-- > 0) {
                if (y >= avctx->height)
                    return AVERROR_INVALIDDATA;

                dst[y * linesize + x] = clr;
                lx = x;
                ly = y;
                x++;
                if (x >= avctx->width) {
                    x = 0;
                    y++;
                }
            }
            break;
        case 1:
            while (run-- > 0) {
                if (y >= avctx->height)
                    return AVERROR_INVALIDDATA;

                dst[y * linesize + x] = dst[ly * linesize + lx];
                lx = x;
                ly = y;
                x++;
                if (x >= avctx->width) {
                    x = 0;
                    y++;
                }
            }
            clr = dst[ly * linesize + lx];
            break;
        case 2:
            while (run-- > 0) {
                if (y < 1 || y >= avctx->height)
                    return AVERROR_INVALIDDATA;

                clr = dst[y * linesize + x + off + 1];
                dst[y * linesize + x] = clr;
                lx = x;
                ly = y;
                x++;
                if (x >= avctx->width) {
                    x = 0;
                    y++;
                }
            }
            break;
        case 4:
            while (run-- > 0) {
                uint8_t *odst = (uint8_t *)dst;

                if (y < 1 || y >= avctx->height ||
                    (y == 1 && x == 0))
                    return AVERROR_INVALIDDATA;

                if (x == 0) {
                    z = backstep;
                } else {
                    z = 0;
                }

                r = odst[(ly * linesize + lx) * 4] +
                    odst[((y * linesize + x) + off - z) * 4 + 4] -
                    odst[((y * linesize + x) + off - z) * 4];
                g = odst[(ly * linesize + lx) * 4 + 1] +
                    odst[((y * linesize + x) + off - z) * 4 + 5] -
                    odst[((y * linesize + x) + off - z) * 4 + 1];
                b = odst[(ly * linesize + lx) * 4 + 2] +
                    odst[((y * linesize + x) + off - z) * 4 + 6] -
                    odst[((y * linesize + x) + off - z) * 4 + 2];
                clr = ((b & 0xFF) << 16) + ((g & 0xFF) << 8) + (r & 0xFF);
                dst[y * linesize + x] = clr;
                lx = x;
                ly = y;
                x++;
                if (x >= avctx->width) {
                    x = 0;
                    y++;
                }
            }
            break;
        case 5:
            while (run-- > 0) {
                if (y < 1 || y >= avctx->height ||
                    (y == 1 && x == 0))
                    return AVERROR_INVALIDDATA;

                if (x == 0) {
                    z = backstep;
                } else {
                    z = 0;
                }

                clr = dst[y * linesize + x + off - z];
                dst[y * linesize + x] = clr;
                lx = x;
                ly = y;
                x++;
                if (x >= avctx->width) {
                    x = 0;
                    y++;
                }
            }
            break;
        }

        if (avctx->bits_per_coded_sample == 16) {
            cx1 = (clr & 0x3F00) >> 2;
            cx = (clr & 0xFFFFFF) >> 16;
        } else {
            cx1 = (clr & 0xFC00) >> 4;
            cx = (clr & 0xFFFFFF) >> 18;
        }
    }

    return 0;
}