Commit message: avcodec/utvideodec: Fix bytes left check in decode_frame()

Fixes: out of array read
Fixes: poc-2017.avi

Found-by: GwanYeong Kim <gy741.kim@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 118e1b0b3370dd1c0da442901b486689efd1654b)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
CVE: None
CWE: CWE-125
Vulnerability description: None

Project: FFmpeg

static int decode_plane10(UtvideoContext *c, int plane_no,
                          uint16_t *dst, int step, int stride,
                          int width, int height,
                          const uint8_t *src, const uint8_t *huff,
                          int use_pred)
{
    int i, j, slice, pix, ret;
    int sstart, send;
    VLC vlc;
    GetBitContext gb;
    int prev, fsym;

    if ((ret = build_huff10(huff, &vlc, &fsym)) < 0) {
        av_log(c->avctx, AV_LOG_ERROR, "Cannot build Huffman codes\n");
        return ret;
    }
    if (fsym >= 0) { // build_huff reported a symbol to fill slices with
        send = 0;
        for (slice = 0; slice < c->slices; slice++) {
            uint16_t *dest;

            sstart = send;
            send   = (height * (slice + 1) / c->slices);
            dest   = dst + sstart * stride;

            prev = 0x200;
            for (j = sstart; j < send; j++) {
                for (i = 0; i < width * step; i += step) {
                    pix = fsym;
                    if (use_pred) {
                        prev += pix;
                        prev &= 0x3FF;
                        pix   = prev;
                    }
                    dest[i] = pix;
                }
                dest += stride;
            }
        }
        return 0;
    }

    send = 0;
    for (slice = 0; slice < c->slices; slice++) {
        uint16_t *dest;
        int slice_data_start, slice_data_end, slice_size;

        sstart = send;
        send   = (height * (slice + 1) / c->slices);
        dest   = dst + sstart * stride;

        // slice offset and size validation was done earlier
        slice_data_start = slice ? AV_RL32(src + slice * 4 - 4) : 0;
        slice_data_end   = AV_RL32(src + slice * 4);
        slice_size       = slice_data_end - slice_data_start;

        if (!slice_size) {
            av_log(c->avctx, AV_LOG_ERROR, "Plane has more than one symbol "
                   "yet a slice has a length of zero.\n");
            goto fail;
        }

        memcpy(c->slice_bits, src + slice_data_start + c->slices * 4,
               slice_size);
        memset(c->slice_bits + slice_size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
        c->bdsp.bswap_buf((uint32_t *) c->slice_bits,
                          (uint32_t *) c->slice_bits,
                          (slice_data_end - slice_data_start + 3) >> 2);
        init_get_bits(&gb, c->slice_bits, slice_size * 8);

        prev = 0x200;
        for (j = sstart; j < send; j++) {
            for (i = 0; i < width * step; i += step) {
                if (get_bits_left(&gb) <= 0) {
                    av_log(c->avctx, AV_LOG_ERROR,
                           "Slice decoding ran out of bits\n");
                    goto fail;
                }
                pix = get_vlc2(&gb, vlc.table, vlc.bits, 3);
                if (pix < 0) {
                    av_log(c->avctx, AV_LOG_ERROR, "Decoding error\n");
                    goto fail;
                }
                if (use_pred) {
                    prev += pix;
                    prev &= 0x3FF;
                    pix   = prev;
                }
                dest[i] = pix;
            }
            dest += stride;
        }
        if (get_bits_left(&gb) > 32)
            av_log(c->avctx, AV_LOG_WARNING,
                   "%d bits left after decoding slice\n", get_bits_left(&gb));
    }

    ff_free_vlc(&vlc);

    return 0;
fail:
    ff_free_vlc(&vlc);
    return AVERROR_INVALIDDATA;
}

Project: FFmpeg

static void restore_rgb_planes10(AVFrame *frame, int width, int height)
{
    uint16_t *src_r = (uint16_t *)frame->data[2];
    uint16_t *src_g = (uint16_t *)frame->data[0];
    uint16_t *src_b = (uint16_t *)frame->data[1];
    int r, g, b;
    int i, j;

    for (j = 0; j < height; j++) {
        for (i = 0; i < width; i++) {
            r = src_r[i];
            g = src_g[i];
            b = src_b[i];
            src_r[i] = (r + g - 0x200) & 0x3FF;
            src_b[i] = (b + g - 0x200) & 0x3FF;
        }
        src_r += frame->linesize[2] / 2;
        src_g += frame->linesize[0] / 2;
        src_b += frame->linesize[1] / 2;
    }
}

Project: FFmpeg

static int build_huff10(const uint8_t *src, VLC *vlc, int *fsym)
{
    int i;
    HuffEntry he[1024];
    int last;
    uint32_t codes[1024];
    uint8_t bits[1024];
    uint16_t syms[1024];
    uint32_t code;

    *fsym = -1;
    for (i = 0; i < 1024; i++) {
        he[i].sym = i;
        he[i].len = *src++;
    }
    qsort(he, 1024, sizeof(*he), ff_ut10_huff_cmp_len);

    if (!he[0].len) {
        *fsym = he[0].sym;
        return 0;
    }

    last = 1023;
    while (he[last].len == 255 && last)
        last--;

    if (he[last].len > 32) {
        return -1;
    }

    code = 1;
    for (i = last; i >= 0; i--) {
        codes[i] = code >> (32 - he[i].len);
        bits[i]  = he[i].len;
        syms[i]  = he[i].sym;
        code += 0x80000000u >> (he[i].len - 1);
    }

    return ff_init_vlc_sparse(vlc, FFMIN(he[last].len, 11), last + 1,
                              bits,  sizeof(*bits),  sizeof(*bits),
                              codes, sizeof(*codes), sizeof(*codes),
                              syms,  sizeof(*syms),  sizeof(*syms), 0);
}

Project: FFmpeg

static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
                        AVPacket *avpkt)
{
    const uint8_t *buf = avpkt->data;
    int buf_size = avpkt->size;
    UtvideoContext *c = avctx->priv_data;
    int i, j;
    const uint8_t *plane_start[5];
    int plane_size, max_slice_size = 0, slice_start, slice_end, slice_size;
    int ret;
    GetByteContext gb;
    ThreadFrame frame = { .f = data };

    if ((ret = ff_thread_get_buffer(avctx, &frame, 0)) < 0)
        return ret;

    /* parse plane structure to get frame flags and validate slice offsets */
    bytestream2_init(&gb, buf, buf_size);
    if (c->pro) {
        if (bytestream2_get_bytes_left(&gb) < c->frame_info_size) {
            av_log(avctx, AV_LOG_ERROR, "Not enough data for frame information\n");
            return AVERROR_INVALIDDATA;
        }
        c->frame_info = bytestream2_get_le32u(&gb);
        c->slices = ((c->frame_info >> 16) & 0xff) + 1;
        for (i = 0; i < c->planes; i++) {
            plane_start[i] = gb.buffer;
            if (bytestream2_get_bytes_left(&gb) < 1024 + 4 * c->slices) {
                av_log(avctx, AV_LOG_ERROR, "Insufficient data for a plane\n");
                return AVERROR_INVALIDDATA;
            }
            slice_start = 0;
            slice_end   = 0;
            for (j = 0; j < c->slices; j++) {
                slice_end   = bytestream2_get_le32u(&gb);
                if (slice_end < 0 || slice_end < slice_start ||
                    bytestream2_get_bytes_left(&gb) < slice_end + 1024LL) {
                    av_log(avctx, AV_LOG_ERROR, "Incorrect slice size\n");
                    return AVERROR_INVALIDDATA;
                }
                slice_size  = slice_end - slice_start;
                slice_start = slice_end;
                max_slice_size = FFMAX(max_slice_size, slice_size);
            }
            plane_size = slice_end;
            bytestream2_skipu(&gb, plane_size);
            bytestream2_skipu(&gb, 1024);
        }
        plane_start[c->planes] = gb.buffer;
    } else {
        for (i = 0; i < c->planes; i++) {
            plane_start[i] = gb.buffer;
            if (bytestream2_get_bytes_left(&gb) < 256 + 4 * c->slices) {
                av_log(avctx, AV_LOG_ERROR, "Insufficient data for a plane\n");
                return AVERROR_INVALIDDATA;
            }
            bytestream2_skipu(&gb, 256);
            slice_start = 0;
            slice_end   = 0;
            for (j = 0; j < c->slices; j++) {
                slice_end   = bytestream2_get_le32u(&gb);
                if (slice_end < 0 || slice_end < slice_start ||
                    bytestream2_get_bytes_left(&gb) < slice_end) {
                    av_log(avctx, AV_LOG_ERROR, "Incorrect slice size\n");
                    return AVERROR_INVALIDDATA;
                }
                slice_size  = slice_end - slice_start;
                slice_start = slice_end;
                max_slice_size = FFMAX(max_slice_size, slice_size);
            }
            plane_size = slice_end;
            bytestream2_skipu(&gb, plane_size);
        }
        plane_start[c->planes] = gb.buffer;
        if (bytestream2_get_bytes_left(&gb) < c->frame_info_size) {
            av_log(avctx, AV_LOG_ERROR, "Not enough data for frame information\n");
            return AVERROR_INVALIDDATA;
        }
        c->frame_info = bytestream2_get_le32u(&gb);
    }
    av_log(avctx, AV_LOG_DEBUG, "frame information flags %"PRIX32"\n",
           c->frame_info);

    c->frame_pred = (c->frame_info >> 8) & 3;

    if (c->frame_pred == PRED_GRADIENT) {
        avpriv_request_sample(avctx, "Frame with gradient prediction");
        return AVERROR_PATCHWELCOME;
    }

    av_fast_malloc(&c->slice_bits, &c->slice_bits_size,
                   max_slice_size + AV_INPUT_BUFFER_PADDING_SIZE);

    if (!c->slice_bits) {
        av_log(avctx, AV_LOG_ERROR, "Cannot allocate temporary buffer\n");
        return AVERROR(ENOMEM);
    }

    switch (c->avctx->pix_fmt) {
    case AV_PIX_FMT_RGB24:
    case AV_PIX_FMT_RGBA:
        for (i = 0; i < c->planes; i++) {
            ret = decode_plane(c, i, frame.f->data[0] + ff_ut_rgb_order[i],
                               c->planes, frame.f->linesize[0], avctx->width,
                               avctx->height, plane_start[i],
                               c->frame_pred == PRED_LEFT);
            if (ret)
                return ret;
            if (c->frame_pred == PRED_MEDIAN) {
                if (!c->interlaced) {
                    restore_median(frame.f->data[0] + ff_ut_rgb_order[i],
                                   c->planes, frame.f->linesize[0], avctx->width,
                                   avctx->height, c->slices, 0);
                } else {
                    restore_median_il(frame.f->data[0] + ff_ut_rgb_order[i],
                                      c->planes, frame.f->linesize[0],
                                      avctx->width, avctx->height, c->slices,
                                      0);
                }
            }
        }
        restore_rgb_planes(frame.f->data[0], c->planes, frame.f->linesize[0],
                           avctx->width, avctx->height);
        break;
    case AV_PIX_FMT_GBRAP10:
    case AV_PIX_FMT_GBRP10:
        for (i = 0; i < c->planes; i++) {
            ret = decode_plane10(c, i, (uint16_t *)frame.f->data[i], 1,
                                 frame.f->linesize[i] / 2, avctx->width,
                                 avctx->height, plane_start[i],
                                 plane_start[i + 1] - 1024,
                                 c->frame_pred == PRED_LEFT);
            if (ret)
                return ret;
        }
        restore_rgb_planes10(frame.f, avctx->width, avctx->height);
        break;
    case AV_PIX_FMT_YUV420P:
        for (i = 0; i < 3; i++) {
            ret = decode_plane(c, i, frame.f->data[i], 1, frame.f->linesize[i],
                               avctx->width >> !!i, avctx->height >> !!i,
                               plane_start[i], c->frame_pred == PRED_LEFT);
            if (ret)
                return ret;
            if (c->frame_pred == PRED_MEDIAN) {
                if (!c->interlaced) {
                    restore_median(frame.f->data[i], 1, frame.f->linesize[i],
                                   avctx->width >> !!i, avctx->height >> !!i,
                                   c->slices, !i);
                } else {
                    restore_median_il(frame.f->data[i], 1, frame.f->linesize[i],
                                      avctx->width  >> !!i,
                                      avctx->height >> !!i,
                                      c->slices, !i);
                }
            }
        }
        break;
    case AV_PIX_FMT_YUV422P:
        for (i = 0; i < 3; i++) {
            ret = decode_plane(c, i, frame.f->data[i], 1, frame.f->linesize[i],
                               avctx->width >> !!i, avctx->height,
                               plane_start[i], c->frame_pred == PRED_LEFT);
            if (ret)
                return ret;
            if (c->frame_pred == PRED_MEDIAN) {
                if (!c->interlaced) {
                    restore_median(frame.f->data[i], 1, frame.f->linesize[i],
                                   avctx->width >> !!i, avctx->height,
                                   c->slices, 0);
                } else {
                    restore_median_il(frame.f->data[i], 1, frame.f->linesize[i],
                                      avctx->width >> !!i, avctx->height,
                                      c->slices, 0);
                }
            }
        }
        break;
    case AV_PIX_FMT_YUV444P:
        for (i = 0; i < 3; i++) {
            ret = decode_plane(c, i, frame.f->data[i], 1, frame.f->linesize[i],
                               avctx->width, avctx->height,
                               plane_start[i], c->frame_pred == PRED_LEFT);
            if (ret)
                return ret;
            if (c->frame_pred == PRED_MEDIAN) {
                if (!c->interlaced) {
                    restore_median(frame.f->data[i], 1, frame.f->linesize[i],
                                   avctx->width, avctx->height,
                                   c->slices, 0);
                } else {
                    restore_median_il(frame.f->data[i], 1, frame.f->linesize[i],
                                      avctx->width, avctx->height,
                                      c->slices, 0);
                }
            }
        }
        break;
    case AV_PIX_FMT_YUV422P10:
        for (i = 0; i < 3; i++) {
            ret = decode_plane10(c, i, (uint16_t *)frame.f->data[i], 1, frame.f->linesize[i] / 2,
                                 avctx->width >> !!i, avctx->height,
                                 plane_start[i], plane_start[i + 1] - 1024, c->frame_pred == PRED_LEFT);
            if (ret)
                return ret;
        }
        break;
    }

    frame.f->key_frame = 1;
    frame.f->pict_type = AV_PICTURE_TYPE_I;
    frame.f->interlaced_frame = !!c->interlaced;

    *got_frame = 1;

    /* always report that the buffer was completely consumed */
    return buf_size;
}

Project: FFmpeg

static av_cold int decode_init(AVCodecContext *avctx)
{
    UtvideoContext * const c = avctx->priv_data;

    c->avctx = avctx;

    ff_bswapdsp_init(&c->bdsp);

    if (avctx->extradata_size >= 16) {
        av_log(avctx, AV_LOG_DEBUG, "Encoder version %d.%d.%d.%d\n",
               avctx->extradata[3], avctx->extradata[2],
               avctx->extradata[1], avctx->extradata[0]);
        av_log(avctx, AV_LOG_DEBUG, "Original format %"PRIX32"\n",
               AV_RB32(avctx->extradata + 4));
        c->frame_info_size = AV_RL32(avctx->extradata + 8);
        c->flags           = AV_RL32(avctx->extradata + 12);

        if (c->frame_info_size != 4)
            avpriv_request_sample(avctx, "Frame info not 4 bytes");
        av_log(avctx, AV_LOG_DEBUG, "Encoding parameters %08"PRIX32"\n", c->flags);
        c->slices      = (c->flags >> 24) + 1;
        c->compression = c->flags & 1;
        c->interlaced  = c->flags & 0x800;
    } else if (avctx->extradata_size == 8) {
        av_log(avctx, AV_LOG_DEBUG, "Encoder version %d.%d.%d.%d\n",
               avctx->extradata[3], avctx->extradata[2],
               avctx->extradata[1], avctx->extradata[0]);
        av_log(avctx, AV_LOG_DEBUG, "Original format %"PRIX32"\n",
               AV_RB32(avctx->extradata + 4));
        c->interlaced  = 0;
        c->pro         = 1;
        c->frame_info_size = 4;
    } else {
        av_log(avctx, AV_LOG_ERROR,
               "Insufficient extradata size %d, should be at least 16\n",
               avctx->extradata_size);
        return AVERROR_INVALIDDATA;
    }

    c->slice_bits_size = 0;

    switch (avctx->codec_tag) {
    case MKTAG('U', 'L', 'R', 'G'):
        c->planes      = 3;
        avctx->pix_fmt = AV_PIX_FMT_RGB24;
        break;
    case MKTAG('U', 'L', 'R', 'A'):
        c->planes      = 4;
        avctx->pix_fmt = AV_PIX_FMT_RGBA;
        break;
    case MKTAG('U', 'L', 'Y', '0'):
        c->planes      = 3;
        avctx->pix_fmt = AV_PIX_FMT_YUV420P;
        avctx->colorspace = AVCOL_SPC_BT470BG;
        break;
    case MKTAG('U', 'L', 'Y', '2'):
        c->planes      = 3;
        avctx->pix_fmt = AV_PIX_FMT_YUV422P;
        avctx->colorspace = AVCOL_SPC_BT470BG;
        break;
    case MKTAG('U', 'L', 'Y', '4'):
        c->planes      = 3;
        avctx->pix_fmt = AV_PIX_FMT_YUV444P;
        avctx->colorspace = AVCOL_SPC_BT470BG;
        break;
    case MKTAG('U', 'Q', 'Y', '2'):
        c->planes      = 3;
        avctx->pix_fmt = AV_PIX_FMT_YUV422P10;
        break;
    case MKTAG('U', 'Q', 'R', 'G'):
        c->planes      = 3;
        avctx->pix_fmt = AV_PIX_FMT_GBRP10;
        break;
    case MKTAG('U', 'Q', 'R', 'A'):
        c->planes      = 4;
        avctx->pix_fmt = AV_PIX_FMT_GBRAP10;
        break;
    case MKTAG('U', 'L', 'H', '0'):
        c->planes      = 3;
        avctx->pix_fmt = AV_PIX_FMT_YUV420P;
        avctx->colorspace = AVCOL_SPC_BT709;
        break;
    case MKTAG('U', 'L', 'H', '2'):
        c->planes      = 3;
        avctx->pix_fmt = AV_PIX_FMT_YUV422P;
        avctx->colorspace = AVCOL_SPC_BT709;
        break;
    case MKTAG('U', 'L', 'H', '4'):
        c->planes      = 3;
        avctx->pix_fmt = AV_PIX_FMT_YUV444P;
        avctx->colorspace = AVCOL_SPC_BT709;
        break;
    default:
        av_log(avctx, AV_LOG_ERROR, "Unknown Ut Video FOURCC provided (%08X)\n",
               avctx->codec_tag);
        return AVERROR_INVALIDDATA;
    }

    return 0;
}