Commit message: avcodec/webp: Always set pix_fmt

Fixes: out of array access
Fixes: 1434/clusterfuzz-testcase-minimized-6314998085189632
Fixes: 1435/clusterfuzz-testcase-minimized-6483783723253760

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Reviewed-by: "Ronald S. Bultje" <rsbultje@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
CVE: CVE-2017-9994
CWE: CWE-119
Vulnerability description: libavcodec/webp.c in FFmpeg before 2.8.12, 3.0.x before 3.0.8, 3.1.x before 3.1.8, 3.2.x before 3.2.5, and 3.3.x before 3.3.1 does not ensure that pix_fmt is set, which allows remote attackers to cause a denial of service (heap-based buffer overflow and application crash) or possibly have unspecified other impact via a crafted file, related to the vp8_decode_mb_row_no_filter and pred8x8_128_dc_8_c functions.

Project: FFmpeg
File: vp8.c

int ff_vp8_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
                        AVPacket *avpkt)
{
    return vp78_decode_frame(avctx, data, got_frame, avpkt, IS_VP8);
}

Project: FFmpeg

static void vp78_update_pred16x16_pred8x8_mvc_probabilities(VP8Context *s,
                                                            int mvc_size)
{
    VP56RangeCoder *c = &s->c;
    int i, j;

    if (vp8_rac_get(c))
        for (i = 0; i < 4; i++)
            s->prob->pred16x16[i] = vp8_rac_get_uint(c, 8);
    if (vp8_rac_get(c))
        for (i = 0; i < 3; i++)
            s->prob->pred8x8c[i]  = vp8_rac_get_uint(c, 8);

    for (i = 0; i < 2; i++)
        for (j = 0; j < mvc_size; j++)
            if (vp56_rac_get_prob_branchy(c, vp8_mv_update_prob[i][j]))
                s->prob->mvc[i][j] = vp8_rac_get_nn(c);
}

Project: FFmpeg
File: vp8.c

static int vp7_calculate_mb_offset(int mb_x, int mb_y, int mb_width,
                                   int xoffset, int yoffset, int boundary,
                                   int *edge_x, int *edge_y)
{
    int vwidth = mb_width + 1;
    int new = (mb_y + yoffset) * vwidth + mb_x + xoffset;
    if (new < boundary || new % vwidth == vwidth - 1)
        return 0;
    *edge_y = new / vwidth;
    *edge_x = new % vwidth;
    return 1;
}

Project: FFmpeg
File: vp8.c

static int vp7_fade_frame(VP8Context *s, VP56RangeCoder *c)
{
    int alpha = (int8_t) vp8_rac_get_uint(c, 8);
    int beta  = (int8_t) vp8_rac_get_uint(c, 8);
    int ret;

    if (!s->keyframe && (alpha || beta)) {
        int width  = s->mb_width * 16;
        int height = s->mb_height * 16;
        AVFrame *src, *dst;

        if (!s->framep[VP56_FRAME_PREVIOUS] ||
            !s->framep[VP56_FRAME_GOLDEN]) {
            av_log(s->avctx, AV_LOG_WARNING, "Discarding interframe without a prior keyframe!\n");
            return AVERROR_INVALIDDATA;
        }

        dst =
        src = s->framep[VP56_FRAME_PREVIOUS]->tf.f;

        /* preserve the golden frame, write a new previous frame */
        if (s->framep[VP56_FRAME_GOLDEN] == s->framep[VP56_FRAME_PREVIOUS]) {
            s->framep[VP56_FRAME_PREVIOUS] = vp8_find_free_buffer(s);
            if ((ret = vp8_alloc_frame(s, s->framep[VP56_FRAME_PREVIOUS], 1)) < 0)
                return ret;

            dst = s->framep[VP56_FRAME_PREVIOUS]->tf.f;

            copy_chroma(dst, src, width, height);
        }

        fade(dst->data[0], dst->linesize[0],
             src->data[0], src->linesize[0],
             width, height, alpha, beta);
    }

    return 0;
}

Project: FFmpeg
File: vp8.c

static int vp8_decode_block_coeffs_internal(VP56RangeCoder *r,
                                            int16_t block[16],
                                            uint8_t probs[16][3][NUM_DCT_TOKENS - 1],
                                            int i, uint8_t *token_prob,
                                            int16_t qmul[2])
{
    return decode_block_coeffs_internal(r, block, probs, i,
                                        token_prob, qmul, ff_zigzag_scan, IS_VP8);
}

Project: FFmpeg
File: vp8.c

static void vp8_decode_flush_impl(AVCodecContext *avctx, int free_mem)
{
    VP8Context *s = avctx->priv_data;
    int i;

    for (i = 0; i < FF_ARRAY_ELEMS(s->frames); i++)
        vp8_release_frame(s, &s->frames[i]);
    memset(s->framep, 0, sizeof(s->framep));

    if (free_mem)
        free_buffers(s);
}

Project: FFmpeg
File: vp8.c

static int vp8_decode_mb_row_no_filter(AVCodecContext *avctx, void *tdata,
                                        int jobnr, int threadnr)
{
    return decode_mb_row_no_filter(avctx, tdata, jobnr, threadnr, 0);
}

Project: FFmpeg
File: vp8.c

static av_cold int vp8_init_frames(VP8Context *s)
{
    int i;
    for (i = 0; i < FF_ARRAY_ELEMS(s->frames); i++) {
        s->frames[i].tf.f = av_frame_alloc();
        if (!s->frames[i].tf.f)
            return AVERROR(ENOMEM);
    }
    return 0;
}

Project: FFmpeg

void vp8_mc_chroma(VP8Context *s, VP8ThreadData *td, uint8_t *dst1,
                   uint8_t *dst2, ThreadFrame *ref, const VP56mv *mv,
                   int x_off, int y_off, int block_w, int block_h,
                   int width, int height, ptrdiff_t linesize,
                   vp8_mc_func mc_func[3][3])
{
    uint8_t *src1 = ref->f->data[1], *src2 = ref->f->data[2];

    if (AV_RN32A(mv)) {
        int mx = mv->x & 7, mx_idx = subpel_idx[0][mx];
        int my = mv->y & 7, my_idx = subpel_idx[0][my];

        x_off += mv->x >> 3;
        y_off += mv->y >> 3;

        src1 += y_off * linesize + x_off;
        src2 += y_off * linesize + x_off;
        ff_thread_await_progress(ref, (3 + y_off + block_h + subpel_idx[2][my]) >> 3, 0);
        if (x_off < mx_idx || x_off >= width  - block_w - subpel_idx[2][mx] ||
            y_off < my_idx || y_off >= height - block_h - subpel_idx[2][my]) {
            s->vdsp.emulated_edge_mc(td->edge_emu_buffer,
                                     src1 - my_idx * linesize - mx_idx,
                                     EDGE_EMU_LINESIZE, linesize,
                                     block_w + subpel_idx[1][mx],
                                     block_h + subpel_idx[1][my],
                                     x_off - mx_idx, y_off - my_idx, width, height);
            src1 = td->edge_emu_buffer + mx_idx + EDGE_EMU_LINESIZE * my_idx;
            mc_func[my_idx][mx_idx](dst1, linesize, src1, EDGE_EMU_LINESIZE, block_h, mx, my);

            s->vdsp.emulated_edge_mc(td->edge_emu_buffer,
                                     src2 - my_idx * linesize - mx_idx,
                                     EDGE_EMU_LINESIZE, linesize,
                                     block_w + subpel_idx[1][mx],
                                     block_h + subpel_idx[1][my],
                                     x_off - mx_idx, y_off - my_idx, width, height);
            src2 = td->edge_emu_buffer + mx_idx + EDGE_EMU_LINESIZE * my_idx;
            mc_func[my_idx][mx_idx](dst2, linesize, src2, EDGE_EMU_LINESIZE, block_h, mx, my);
        } else {
            mc_func[my_idx][mx_idx](dst1, linesize, src1, linesize, block_h, mx, my);
            mc_func[my_idx][mx_idx](dst2, linesize, src2, linesize, block_h, mx, my);
        }
    } else {
        ff_thread_await_progress(ref, (3 + y_off + block_h) >> 3, 0);
        mc_func[0][0](dst1, linesize, src1 + y_off * linesize + x_off, linesize, block_h, 0, 0);
        mc_func[0][0](dst2, linesize, src2 + y_off * linesize + x_off, linesize, block_h, 0, 0);
    }
}

Project: FFmpeg
File: vp8.c

void vp8_mc_part(VP8Context *s, VP8ThreadData *td, uint8_t *dst[3],
                 ThreadFrame *ref_frame, int x_off, int y_off,
                 int bx_off, int by_off, int block_w, int block_h,
                 int width, int height, VP56mv *mv)
{
    VP56mv uvmv = *mv;

    /* Y */
    vp8_mc_luma(s, td, dst[0] + by_off * s->linesize + bx_off,
                ref_frame, mv, x_off + bx_off, y_off + by_off,
                block_w, block_h, width, height, s->linesize,
                s->put_pixels_tab[block_w == 8]);

    /* U/V */
    if (s->profile == 3) {
        /* this block only applies VP8; it is safe to check
         * only the profile, as VP7 profile <= 1 */
        uvmv.x &= ~7;
        uvmv.y &= ~7;
    }
    x_off   >>= 1;
    y_off   >>= 1;
    bx_off  >>= 1;
    by_off  >>= 1;
    width   >>= 1;
    height  >>= 1;
    block_w >>= 1;
    block_h >>= 1;
    vp8_mc_chroma(s, td, dst[1] + by_off * s->uvlinesize + bx_off,
                  dst[2] + by_off * s->uvlinesize + bx_off, ref_frame,
                  &uvmv, x_off + bx_off, y_off + by_off,
                  block_w, block_h, width, height, s->uvlinesize,
                  s->put_pixels_tab[1 + (block_w == 4)]);
}

Project: FFmpeg
File: webp.c

static int apply_predictor_transform(WebPContext *s)
{
    ImageContext *img  = &s->image[IMAGE_ROLE_ARGB];
    ImageContext *pimg = &s->image[IMAGE_ROLE_PREDICTOR];
    int x, y;

    for (y = 0; y < img->frame->height; y++) {
        for (x = 0; x < img->frame->width; x++) {
            int tx = x >> pimg->size_reduction;
            int ty = y >> pimg->size_reduction;
            enum PredictionMode m = GET_PIXEL_COMP(pimg->frame, tx, ty, 2);

            if (x == 0) {
                if (y == 0)
                    m = PRED_MODE_BLACK;
                else
                    m = PRED_MODE_T;
            } else if (y == 0)
                m = PRED_MODE_L;

            if (m > 13) {
                av_log(s->avctx, AV_LOG_ERROR,
                       "invalid predictor mode: %d\n", m);
                return AVERROR_INVALIDDATA;
            }
            inverse_prediction(img->frame, m, x, y);
        }
    }
    return 0;
}

Project: FFmpeg
File: webp.c

static HuffReader *get_huffman_group(WebPContext *s, ImageContext *img,
                                     int x, int y)
{
    ImageContext *gimg = &s->image[IMAGE_ROLE_ENTROPY];
    int group = 0;

    if (gimg->size_reduction > 0) {
        int group_x = x >> gimg->size_reduction;
        int group_y = y >> gimg->size_reduction;
        int g0      = GET_PIXEL_COMP(gimg->frame, group_x, group_y, 1);
        int g1      = GET_PIXEL_COMP(gimg->frame, group_x, group_y, 2);
        group       = g0 << 8 | g1;
    }

    return &img->huffman_groups[group * HUFFMAN_CODES_PER_META_CODE];
}

Project: FFmpeg
File: webp.c

static void inv_predict_0(uint8_t *p, const uint8_t *p_l, const uint8_t *p_tl,
                          const uint8_t *p_t, const uint8_t *p_tr)
{
    AV_WB32(p, 0xFF000000);
}

Project: FFmpeg
File: webp.c

static void inv_predict_12(uint8_t *p, const uint8_t *p_l, const uint8_t *p_tl,
                           const uint8_t *p_t, const uint8_t *p_tr)
{
    p[0] = av_clip_uint8(p_l[0] + p_t[0] - p_tl[0]);
    p[1] = av_clip_uint8(p_l[1] + p_t[1] - p_tl[1]);
    p[2] = av_clip_uint8(p_l[2] + p_t[2] - p_tl[2]);
    p[3] = av_clip_uint8(p_l[3] + p_t[3] - p_tl[3]);
}