Commit message: avcodec/g2meet: Fix framebuf size

Currently the code can in some cases draw tiles that hang outside the
allocated buffer. This patch increases the buffer size to avoid out
of array accesses. An alternative would be to fail if such tiles are
encountered.
I do not know if any valid files use such hanging tiles.

Fixes Ticket2971
Found-by: ami_stuff
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
CVE: CVE-2013-7022
CWE: CWE-119
Vulnerability description: The g2m_init_buffers function in libavcodec/g2meet.c in FFmpeg before 2.1 does not properly allocate memory for tiles, which allows remote attackers to cause a denial of service (out-of-bounds array access) or possibly have unspecified other impact via crafted Go2Webinar data.

Project: FFmpeg
File: g2meet.c

static av_cold int build_vlc(VLC *vlc, const uint8_t *bits_table,
                             const uint8_t *val_table, int nb_codes,
                             int is_ac)
{
    uint8_t  huff_size[256] = { 0 };
    uint16_t huff_code[256];
    uint16_t huff_sym[256];
    int i;

    ff_mjpeg_build_huffman_codes(huff_size, huff_code, bits_table, val_table);

    for (i = 0; i < 256; i++)
        huff_sym[i] = i + 16 * is_ac;

    if (is_ac)
        huff_sym[0] = 16 * 256;

    return ff_init_vlc_sparse(vlc, 9, nb_codes, huff_size, 1, 1,
                              huff_code, 2, 2, huff_sym, 2, 2, 0);
}

Project: FFmpeg
File: g2meet.c

static av_cold int g2m_decode_end(AVCodecContext *avctx)
{
    G2MContext * const c = avctx->priv_data;

    jpg_free_context(&c->jc);

    av_freep(&c->kempf_buf);
    av_freep(&c->kempf_flags);
    av_freep(&c->synth_tile);
    av_freep(&c->jpeg_tile);
    av_freep(&c->cursor);
    av_freep(&c->framebuf);

    return 0;
}

Project: FFmpeg
File: g2meet.c

static int g2m_decode_frame(AVCodecContext *avctx, void *data,
                            int *got_picture_ptr, AVPacket *avpkt)
{
    const uint8_t *buf = avpkt->data;
    int buf_size = avpkt->size;
    G2MContext *c = avctx->priv_data;
    AVFrame *pic = data;
    GetByteContext bc, tbc;
    int magic;
    int got_header = 0;
    uint32_t chunk_size;
    int chunk_type;
    int i;
    int ret;

    if (buf_size < 12) {
        av_log(avctx, AV_LOG_ERROR,
               "Frame should have at least 12 bytes, got %d instead\n",
               buf_size);
        return AVERROR_INVALIDDATA;
    }

    bytestream2_init(&bc, buf, buf_size);

    magic = bytestream2_get_be32(&bc);
    if ((magic & ~0xF) != MKBETAG('G', '2', 'M', '0') ||
        (magic & 0xF) < 2 || (magic & 0xF) > 4) {
        av_log(avctx, AV_LOG_ERROR, "Wrong magic %08X\n", magic);
        return AVERROR_INVALIDDATA;
    }

    if ((magic & 0xF) != 4) {
        av_log(avctx, AV_LOG_ERROR, "G2M2 and G2M3 are not yet supported\n");
        return AVERROR(ENOSYS);
    }

    while (bytestream2_get_bytes_left(&bc) > 5) {
        chunk_size = bytestream2_get_le32(&bc) - 1;
        chunk_type = bytestream2_get_byte(&bc);
        if (chunk_size > bytestream2_get_bytes_left(&bc)) {
            av_log(avctx, AV_LOG_ERROR, "Invalid chunk size %d type %02X\n",
                   chunk_size, chunk_type);
            break;
        }
        switch (chunk_type) {
        case FRAME_INFO:
            c->got_header = 0;
            if (chunk_size < 21) {
                av_log(avctx, AV_LOG_ERROR, "Invalid frame info size %d\n",
                       chunk_size);
                break;
            }
            c->width  = bytestream2_get_be32(&bc);
            c->height = bytestream2_get_be32(&bc);
            if (c->width  < 16 || c->width  > avctx->width ||
                c->height < 16 || c->height > avctx->height) {
                av_log(avctx, AV_LOG_ERROR,
                       "Invalid frame dimensions %dx%d\n",
                       c->width, c->height);
                ret = AVERROR_INVALIDDATA;
                goto header_fail;
            }
            if (c->width != avctx->width || c->height != avctx->height)
                avcodec_set_dimensions(avctx, c->width, c->height);
            c->compression = bytestream2_get_be32(&bc);
            if (c->compression != 2 && c->compression != 3) {
                av_log(avctx, AV_LOG_ERROR,
                       "Unknown compression method %d\n",
                       c->compression);
                return AVERROR_PATCHWELCOME;
            }
            c->tile_width  = bytestream2_get_be32(&bc);
            c->tile_height = bytestream2_get_be32(&bc);
            if (!c->tile_width || !c->tile_height) {
                av_log(avctx, AV_LOG_ERROR,
                       "Invalid tile dimensions %dx%d\n",
                       c->tile_width, c->tile_height);
                ret = AVERROR_INVALIDDATA;
                goto header_fail;
            }
            c->tiles_x = (c->width  + c->tile_width  - 1) / c->tile_width;
            c->tiles_y = (c->height + c->tile_height - 1) / c->tile_height;
            c->bpp = bytestream2_get_byte(&bc);
            chunk_size -= 21;
            bytestream2_skip(&bc, chunk_size);
            if (g2m_init_buffers(c)) {
                ret = AVERROR(ENOMEM);
                goto header_fail;
            }
            got_header = 1;
            break;
        case TILE_DATA:
            if (!c->tiles_x || !c->tiles_y) {
                av_log(avctx, AV_LOG_WARNING,
                       "No frame header - skipping tile\n");
                bytestream2_skip(&bc, bytestream2_get_bytes_left(&bc));
                break;
            }
            if (chunk_size < 2) {
                av_log(avctx, AV_LOG_ERROR, "Invalid tile data size %d\n",
                       chunk_size);
                break;
            }
            c->tile_x = bytestream2_get_byte(&bc);
            c->tile_y = bytestream2_get_byte(&bc);
            if (c->tile_x >= c->tiles_x || c->tile_y >= c->tiles_y) {
                av_log(avctx, AV_LOG_ERROR,
                       "Invalid tile pos %d,%d (in %dx%d grid)\n",
                       c->tile_x, c->tile_y, c->tiles_x, c->tiles_y);
                break;
            }
            chunk_size -= 2;
            ret = 0;
            switch (c->compression) {
            case COMPR_EPIC_J_B:
                av_log(avctx, AV_LOG_ERROR,
                       "ePIC j-b compression is not implemented yet\n");
                return AVERROR(ENOSYS);
            case COMPR_KEMPF_J_B:
                ret = kempf_decode_tile(c, c->tile_x, c->tile_y,
                                        buf + bytestream2_tell(&bc),
                                        chunk_size);
                break;
            }
            if (ret && c->framebuf)
                av_log(avctx, AV_LOG_ERROR, "Error decoding tile %d,%d\n",
                       c->tile_x, c->tile_y);
            bytestream2_skip(&bc, chunk_size);
            break;
        case CURSOR_POS:
            if (chunk_size < 5) {
                av_log(avctx, AV_LOG_ERROR, "Invalid cursor pos size %d\n",
                       chunk_size);
                break;
            }
            c->cursor_x = bytestream2_get_be16(&bc);
            c->cursor_y = bytestream2_get_be16(&bc);
            bytestream2_skip(&bc, chunk_size - 4);
            break;
        case CURSOR_SHAPE:
            if (chunk_size < 8) {
                av_log(avctx, AV_LOG_ERROR, "Invalid cursor data size %d\n",
                       chunk_size);
                break;
            }
            bytestream2_init(&tbc, buf + bytestream2_tell(&bc),
                             chunk_size - 4);
            g2m_load_cursor(avctx, c, &tbc);
            bytestream2_skip(&bc, chunk_size);
            break;
        case CHUNK_CC:
        case CHUNK_CD:
            bytestream2_skip(&bc, chunk_size);
            break;
        default:
            av_log(avctx, AV_LOG_WARNING, "Skipping chunk type %02X\n",
                   chunk_type);
            bytestream2_skip(&bc, chunk_size);
        }
    }
    if (got_header)
        c->got_header = 1;

    if (c->width && c->height && c->framebuf) {
        if ((ret = ff_get_buffer(avctx, pic, 0)) < 0)
            return ret;

        pic->key_frame = got_header;
        pic->pict_type = got_header ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_P;

        for (i = 0; i < avctx->height; i++)
            memcpy(pic->data[0] + i * pic->linesize[0],
                   c->framebuf  + i * c->framebuf_stride,
                   c->width * 3);
        g2m_paint_cursor(c, pic->data[0], pic->linesize[0]);

        *got_picture_ptr = 1;
    }

    return buf_size;
header_fail:
    c->width   = c->height  = 0;
    c->tiles_x = c->tiles_y = 0;
    return ret;
}

Project: FFmpeg
File: g2meet.c

static av_cold int g2m_decode_init(AVCodecContext *avctx)
{
    G2MContext * const c = avctx->priv_data;
    int ret;

    if ((ret = jpg_init(avctx, &c->jc)) != 0) {
        av_log(avctx, AV_LOG_ERROR, "Cannot initialise VLCs\n");
        jpg_free_context(&c->jc);
        return AVERROR(ENOMEM);
    }

    avctx->pix_fmt = AV_PIX_FMT_RGB24;

    return 0;
}

Project: FFmpeg
File: g2meet.c

static int g2m_load_cursor(AVCodecContext *avctx, G2MContext *c,
                           GetByteContext *gb)
{
    int i, j, k;
    uint8_t *dst;
    uint32_t bits;
    uint32_t cur_size, cursor_w, cursor_h, cursor_stride;
    uint32_t cursor_hot_x, cursor_hot_y;
    int cursor_fmt;
    uint8_t *tmp;

    cur_size      = bytestream2_get_be32(gb);
    cursor_w      = bytestream2_get_byte(gb);
    cursor_h      = bytestream2_get_byte(gb);
    cursor_hot_x  = bytestream2_get_byte(gb);
    cursor_hot_y  = bytestream2_get_byte(gb);
    cursor_fmt    = bytestream2_get_byte(gb);

    cursor_stride = FFALIGN(cursor_w, 32) * 4;

    if (cursor_w < 1 || cursor_w > 256 ||
        cursor_h < 1 || cursor_h > 256) {
        av_log(avctx, AV_LOG_ERROR, "Invalid cursor dimensions %dx%d\n",
               cursor_w, cursor_h);
        return AVERROR_INVALIDDATA;
    }
    if (cursor_hot_x > cursor_w || cursor_hot_y > cursor_h) {
        av_log(avctx, AV_LOG_WARNING, "Invalid hotspot position %d,%d\n",
               cursor_hot_x, cursor_hot_y);
        cursor_hot_x = FFMIN(cursor_hot_x, cursor_w - 1);
        cursor_hot_y = FFMIN(cursor_hot_y, cursor_h - 1);
    }
    if (cur_size - 9 > bytestream2_get_bytes_left(gb) ||
        c->cursor_w * c->cursor_h / 4 > cur_size) {
        av_log(avctx, AV_LOG_ERROR, "Invalid cursor data size %d/%d\n",
               cur_size, bytestream2_get_bytes_left(gb));
        return AVERROR_INVALIDDATA;
    }
    if (cursor_fmt != 1 && cursor_fmt != 32) {
        avpriv_report_missing_feature(avctx, "Cursor format %d",
                                      cursor_fmt);
        return AVERROR_PATCHWELCOME;
    }

    tmp = av_realloc(c->cursor, cursor_stride * cursor_h);
    if (!tmp) {
        av_log(avctx, AV_LOG_ERROR, "Cannot allocate cursor buffer\n");
        return AVERROR(ENOMEM);
    }

    c->cursor        = tmp;
    c->cursor_w      = cursor_w;
    c->cursor_h      = cursor_h;
    c->cursor_hot_x  = cursor_hot_x;
    c->cursor_hot_y  = cursor_hot_y;
    c->cursor_fmt    = cursor_fmt;
    c->cursor_stride = cursor_stride;

    dst = c->cursor;
    switch (c->cursor_fmt) {
    case 1: // old monochrome
        for (j = 0; j < c->cursor_h; j++) {
            for (i = 0; i < c->cursor_w; i += 32) {
                bits = bytestream2_get_be32(gb);
                for (k = 0; k < 32; k++) {
                    dst[0] = !!(bits & 0x80000000);
                    dst += 4;
                    bits <<= 1;
                }
            }
        }

        dst = c->cursor;
        for (j = 0; j < c->cursor_h; j++) {
            for (i = 0; i < c->cursor_w; i += 32) {
                bits = bytestream2_get_be32(gb);
                for (k = 0; k < 32; k++) {
                    int mask_bit = !!(bits & 0x80000000);
                    switch (dst[0] * 2 + mask_bit) {
                    case 0:
                        dst[0] = 0xFF; dst[1] = 0x00;
                        dst[2] = 0x00; dst[3] = 0x00;
                        break;
                    case 1:
                        dst[0] = 0xFF; dst[1] = 0xFF;
                        dst[2] = 0xFF; dst[3] = 0xFF;
                        break;
                    default:
                        dst[0] = 0x00; dst[1] = 0x00;
                        dst[2] = 0x00; dst[3] = 0x00;
                    }
                    dst += 4;
                    bits <<= 1;
                }
            }
        }
        break;
    case 32: // full colour
        /* skip monochrome version of the cursor and decode RGBA instead */
        bytestream2_skip(gb, c->cursor_h * (FFALIGN(c->cursor_w, 32) >> 3));
        for (j = 0; j < c->cursor_h; j++) {
            for (i = 0; i < c->cursor_w; i++) {
                int val = bytestream2_get_be32(gb);
                *dst++ = val >>  0;
                *dst++ = val >>  8;
                *dst++ = val >> 16;
                *dst++ = val >> 24;
            }
        }
        break;
    default:
        return AVERROR_PATCHWELCOME;
    }
    return 0;
}

Project: FFmpeg
File: g2meet.c

static void g2m_paint_cursor(G2MContext *c, uint8_t *dst, int stride)
{
    int i, j;
    int x, y, w, h;
    const uint8_t *cursor;

    if (!c->cursor)
        return;

    x = c->cursor_x - c->cursor_hot_x;
    y = c->cursor_y - c->cursor_hot_y;

    cursor = c->cursor;
    w      = c->cursor_w;
    h      = c->cursor_h;

    if (x + w > c->width)
        w = c->width - x;
    if (y + h > c->height)
        h = c->height - y;
    if (x < 0) {
        w      +=  x;
        cursor += -x * 4;
    } else {
        dst    +=  x * 3;
    }
    if (y < 0) {
        h      +=  y;
        cursor += -y * c->cursor_stride;
    } else {
        dst    +=  y * stride;
    }
    if (w < 0 || h < 0)
        return;

    for (j = 0; j < h; j++) {
        for (i = 0; i < w; i++) {
            uint8_t alpha = cursor[i * 4];
            APPLY_ALPHA(dst[i * 3 + 0], cursor[i * 4 + 1], alpha);
            APPLY_ALPHA(dst[i * 3 + 1], cursor[i * 4 + 2], alpha);
            APPLY_ALPHA(dst[i * 3 + 2], cursor[i * 4 + 3], alpha);
        }
        dst    += stride;
        cursor += c->cursor_stride;
    }
}

Project: FFmpeg
File: g2meet.c

static av_cold void jpg_free_context(JPGContext *ctx)
{
    int i;

    for (i = 0; i < 2; i++) {
        ff_free_vlc(&ctx->dc_vlc[i]);
        ff_free_vlc(&ctx->ac_vlc[i]);
    }

    av_freep(&ctx->buf);
}

Project: FFmpeg
File: g2meet.c

static av_cold int jpg_init(AVCodecContext *avctx, JPGContext *c)
{
    int ret;

    ret = build_vlc(&c->dc_vlc[0], avpriv_mjpeg_bits_dc_luminance,
                    avpriv_mjpeg_val_dc, 12, 0);
    if (ret)
        return ret;
    ret = build_vlc(&c->dc_vlc[1], avpriv_mjpeg_bits_dc_chrominance,
                    avpriv_mjpeg_val_dc, 12, 0);
    if (ret)
        return ret;
    ret = build_vlc(&c->ac_vlc[0], avpriv_mjpeg_bits_ac_luminance,
                    avpriv_mjpeg_val_ac_luminance, 251, 1);
    if (ret)
        return ret;
    ret = build_vlc(&c->ac_vlc[1], avpriv_mjpeg_bits_ac_chrominance,
                    avpriv_mjpeg_val_ac_chrominance, 251, 1);
    if (ret)
        return ret;

    ff_dsputil_init(&c->dsp, avctx);
    ff_init_scantable(c->dsp.idct_permutation, &c->scantable,
                      ff_zigzag_direct);

    return 0;
}

Project: FFmpeg
File: g2meet.c

static void jpg_unescape(const uint8_t *src, int src_size,
                         uint8_t *dst, int *dst_size)
{
    const uint8_t *src_end = src + src_size;
    uint8_t *dst_start = dst;

    while (src < src_end) {
        uint8_t x = *src++;

        *dst++ = x;

        if (x == 0xFF && !*src)
            src++;
    }
    *dst_size = dst - dst_start;
}

Project: FFmpeg
File: g2meet.c

static void kempf_restore_buf(const uint8_t *src, int len,
                              uint8_t *dst, int stride,
                              const uint8_t *jpeg_tile, int tile_stride,
                              int width, int height,
                              const uint8_t *pal, int npal, int tidx)
{
    GetBitContext gb;
    int i, j, nb, col;

    init_get_bits(&gb, src, len * 8);

    if (npal <= 2)       nb = 1;
    else if (npal <= 4)  nb = 2;
    else if (npal <= 16) nb = 4;
    else                 nb = 8;

    for (j = 0; j < height; j++, dst += stride, jpeg_tile += tile_stride) {
        if (get_bits(&gb, 8))
            continue;
        for (i = 0; i < width; i++) {
            col = get_bits(&gb, nb);
            if (col != tidx)
                memcpy(dst + i * 3, pal + col * 3, 3);
            else
                memcpy(dst + i * 3, jpeg_tile + i * 3, 3);
        }
    }
}

Project: FFmpeg
File: g2meet.c

static inline void yuv2rgb(uint8_t *out, int Y, int U, int V)
{
    out[0] = av_clip_uint8(Y + (             91881 * V + 32768 >> 16));
    out[1] = av_clip_uint8(Y + (-22554 * U - 46802 * V + 32768 >> 16));
    out[2] = av_clip_uint8(Y + (116130 * U             + 32768 >> 16));
}