Commit message: avcodec/dnxhddec: Move mb height check out of non hr branch

Fixes: out of array access
Fixes: poc.dnxhd

Found-by: Bingchang, Liu@VARAS of IIE
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 296debd213bd6dce7647cedd34eb64e5b94cdc92)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
CVE: None
CWE: CWE-125
Vulnerability description: None

Project: FFmpeg
File: dnxhddec.c

static int dnxhd_decode_dct_block_12(const DNXHDContext *ctx,
                                     RowContext *row, int n)
{
    return dnxhd_decode_dct_block(ctx, row, n, 6, 8, 4, 2);
}

Project: FFmpeg
File: dnxhddec.c

static int dnxhd_init_vlc(DNXHDContext *ctx, uint32_t cid, int bitdepth)
{
    if (cid != ctx->cid) {
        int index;

        if ((index = ff_dnxhd_get_cid_table(cid)) < 0) {
            av_log(ctx->avctx, AV_LOG_ERROR, "unsupported cid %d\n", cid);
            return AVERROR(ENOSYS);
        }
        if (ff_dnxhd_cid_table[index].bit_depth != bitdepth &&
            ff_dnxhd_cid_table[index].bit_depth != DNXHD_VARIABLE) {
            av_log(ctx->avctx, AV_LOG_ERROR, "bit depth mismatches %d %d\n", ff_dnxhd_cid_table[index].bit_depth, bitdepth);
            return AVERROR_INVALIDDATA;
        }
        if (bitdepth > 10) {
            avpriv_request_sample(ctx->avctx, "DNXHR 12-bit");
            if (ctx->avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL)
                return AVERROR_PATCHWELCOME;
        }
        ctx->cid_table = &ff_dnxhd_cid_table[index];
        av_log(ctx->avctx, AV_LOG_VERBOSE, "Profile cid %d.\n", cid);

        ff_free_vlc(&ctx->ac_vlc);
        ff_free_vlc(&ctx->dc_vlc);
        ff_free_vlc(&ctx->run_vlc);

        init_vlc(&ctx->ac_vlc, DNXHD_VLC_BITS, 257,
                 ctx->cid_table->ac_bits, 1, 1,
                 ctx->cid_table->ac_codes, 2, 2, 0);
        init_vlc(&ctx->dc_vlc, DNXHD_DC_VLC_BITS, bitdepth + 4,
                 ctx->cid_table->dc_bits, 1, 1,
                 ctx->cid_table->dc_codes, 1, 1, 0);
        init_vlc(&ctx->run_vlc, DNXHD_VLC_BITS, 62,
                 ctx->cid_table->run_bits, 1, 1,
                 ctx->cid_table->run_codes, 2, 2, 0);

        ctx->cid = cid;
    }
    return 0;
}

Project: FFmpeg
File: dnxhddec.c

static int dnxhd_decode_header(DNXHDContext *ctx, AVFrame *frame,
                               const uint8_t *buf, int buf_size,
                               int first_field)
{
    int i, cid, ret;
    int old_bit_depth = ctx->bit_depth, bitdepth;
    uint64_t header_prefix;
    if (buf_size < 0x280) {
        av_log(ctx->avctx, AV_LOG_ERROR,
               "buffer too small (%d < 640).\n", buf_size);
        return AVERROR_INVALIDDATA;
    }

    header_prefix = avpriv_dnxhd_parse_header_prefix(buf);
    if (header_prefix == 0) {
        av_log(ctx->avctx, AV_LOG_ERROR,
               "unknown header 0x%02X 0x%02X 0x%02X 0x%02X 0x%02X\n",
               buf[0], buf[1], buf[2], buf[3], buf[4]);
        return AVERROR_INVALIDDATA;
    }
    if (buf[5] & 2) { /* interlaced */
        ctx->cur_field = buf[5] & 1;
        frame->interlaced_frame = 1;
        frame->top_field_first  = first_field ^ ctx->cur_field;
        av_log(ctx->avctx, AV_LOG_DEBUG,
               "interlaced %d, cur field %d\n", buf[5] & 3, ctx->cur_field);
    } else {
        ctx->cur_field = 0;
    }
    ctx->mbaff = (buf[0x6] >> 5) & 1;

    ctx->height = AV_RB16(buf + 0x18);
    ctx->width  = AV_RB16(buf + 0x1a);

    switch(buf[0x21] >> 5) {
    case 1: bitdepth = 8; break;
    case 2: bitdepth = 10; break;
    case 3: bitdepth = 12; break;
    default:
        av_log(ctx->avctx, AV_LOG_ERROR,
               "Unknown bitdepth indicator (%d)\n", buf[0x21] >> 5);
        return AVERROR_INVALIDDATA;
    }

    cid = AV_RB32(buf + 0x28);
    if ((ret = dnxhd_init_vlc(ctx, cid, bitdepth)) < 0)
        return ret;
    if (ctx->mbaff && ctx->cid_table->cid != 1260)
        av_log(ctx->avctx, AV_LOG_WARNING,
               "Adaptive MB interlace flag in an unsupported profile.\n");

    ctx->act = buf[0x2C] & 7;
    if (ctx->act && ctx->cid_table->cid != 1256 && ctx->cid_table->cid != 1270)
        av_log(ctx->avctx, AV_LOG_WARNING,
               "Adaptive color transform in an unsupported profile.\n");

    ctx->is_444 = (buf[0x2C] >> 6) & 1;
    if (ctx->is_444) {
        if (bitdepth == 8) {
            avpriv_request_sample(ctx->avctx, "4:4:4 8 bits\n");
            return AVERROR_INVALIDDATA;
        } else if (bitdepth == 10) {
            ctx->decode_dct_block = dnxhd_decode_dct_block_10_444;
            ctx->pix_fmt = ctx->act ? AV_PIX_FMT_YUV444P10
                                    : AV_PIX_FMT_GBRP10;
        } else {
            ctx->decode_dct_block = dnxhd_decode_dct_block_12_444;
            ctx->pix_fmt = ctx->act ? AV_PIX_FMT_YUV444P12
                                    : AV_PIX_FMT_GBRP12;
        }
    } else if (bitdepth == 12) {
        ctx->decode_dct_block = dnxhd_decode_dct_block_12;
        ctx->pix_fmt = AV_PIX_FMT_YUV422P12;
    } else if (bitdepth == 10) {
        ctx->decode_dct_block = dnxhd_decode_dct_block_10;
        ctx->pix_fmt = AV_PIX_FMT_YUV422P10;
    } else {
        ctx->decode_dct_block = dnxhd_decode_dct_block_8;
        ctx->pix_fmt = AV_PIX_FMT_YUV422P;
    }

    ctx->avctx->bits_per_raw_sample = ctx->bit_depth = bitdepth;
    if (ctx->bit_depth != old_bit_depth) {
        ff_blockdsp_init(&ctx->bdsp, ctx->avctx);
        ff_idctdsp_init(&ctx->idsp, ctx->avctx);
        ff_init_scantable(ctx->idsp.idct_permutation, &ctx->scantable,
                          ff_zigzag_direct);
    }

    // make sure profile size constraints are respected
    // DNx100 allows 1920->1440 and 1280->960 subsampling
    if (ctx->width != ctx->cid_table->width &&
        ctx->cid_table->width != DNXHD_VARIABLE) {
        av_reduce(&ctx->avctx->sample_aspect_ratio.num,
                  &ctx->avctx->sample_aspect_ratio.den,
                  ctx->width, ctx->cid_table->width, 255);
        ctx->width = ctx->cid_table->width;
    }

    if (buf_size < ctx->cid_table->coding_unit_size) {
        av_log(ctx->avctx, AV_LOG_ERROR, "incorrect frame size (%d < %d).\n",
               buf_size, ctx->cid_table->coding_unit_size);
        return AVERROR_INVALIDDATA;
    }

    ctx->mb_width  = (ctx->width + 15)>> 4;
    ctx->mb_height = buf[0x16d];

    if ((ctx->height + 15) >> 4 == ctx->mb_height && frame->interlaced_frame)
        ctx->height <<= 1;

    av_log(ctx->avctx, AV_LOG_VERBOSE, "%dx%d, 4:%s %d bits, MBAFF=%d ACT=%d\n",
           ctx->width, ctx->height, ctx->is_444 ? "4:4" : "2:2",
           ctx->bit_depth, ctx->mbaff, ctx->act);

    // Newer format supports variable mb_scan_index sizes
    if (header_prefix == DNXHD_HEADER_HR2) {
        ctx->data_offset = 0x170 + (ctx->mb_height << 2);
    } else {
        if (ctx->mb_height > 68) {
            av_log(ctx->avctx, AV_LOG_ERROR,
                   "mb height too big: %d\n", ctx->mb_height);
            return AVERROR_INVALIDDATA;
        }
        ctx->data_offset = 0x280;
    }
    if ((ctx->mb_height << frame->interlaced_frame) > (ctx->height + 15) >> 4) {
        av_log(ctx->avctx, AV_LOG_ERROR,
                "mb height too big: %d\n", ctx->mb_height);
        return AVERROR_INVALIDDATA;
    }

    if (buf_size < ctx->data_offset) {
        av_log(ctx->avctx, AV_LOG_ERROR,
               "buffer too small (%d < %d).\n", buf_size, ctx->data_offset);
        return AVERROR_INVALIDDATA;
    }

    av_assert0((unsigned)ctx->mb_height <= FF_ARRAY_ELEMS(ctx->mb_scan_index));

    for (i = 0; i < ctx->mb_height; i++) {
        ctx->mb_scan_index[i] = AV_RB32(buf + 0x170 + (i << 2));
        ff_dlog(ctx->avctx, "mb scan index %d, pos %d: %u\n", i, 0x170 + (i << 2), ctx->mb_scan_index[i]);
        if (buf_size - ctx->data_offset < ctx->mb_scan_index[i]) {
            av_log(ctx->avctx, AV_LOG_ERROR,
                   "invalid mb scan index (%u vs %u).\n",
                   ctx->mb_scan_index[i], buf_size - ctx->data_offset);
            return AVERROR_INVALIDDATA;
        }
    }

    return 0;
}

Project: FFmpeg
File: dnxhddec.c

static av_cold int dnxhd_decode_init_thread_copy(AVCodecContext *avctx)
{
    DNXHDContext *ctx = avctx->priv_data;

    // make sure VLC tables will be loaded when cid is parsed
    ctx->cid = -1;

    ctx->rows = av_mallocz_array(avctx->thread_count, sizeof(RowContext));
    if (!ctx->rows)
        return AVERROR(ENOMEM);

    return 0;
}

Project: FFmpeg
File: dnxhddec.c

static int dnxhd_decode_frame(AVCodecContext *avctx, void *data,
                              int *got_frame, AVPacket *avpkt)
{
    const uint8_t *buf = avpkt->data;
    int buf_size = avpkt->size;
    DNXHDContext *ctx = avctx->priv_data;
    ThreadFrame frame = { .f = data };
    AVFrame *picture = data;
    int first_field = 1;
    int ret, i;

    ff_dlog(avctx, "frame size %d\n", buf_size);

    for (i = 0; i < avctx->thread_count; i++)
        ctx->rows[i].format = -1;

decode_coding_unit:
    if ((ret = dnxhd_decode_header(ctx, picture, buf, buf_size, first_field)) < 0)
        return ret;

    if ((avctx->width || avctx->height) &&
        (ctx->width != avctx->width || ctx->height != avctx->height)) {
        av_log(avctx, AV_LOG_WARNING, "frame size changed: %dx%d -> %dx%d\n",
               avctx->width, avctx->height, ctx->width, ctx->height);
        first_field = 1;
    }
    if (avctx->pix_fmt != AV_PIX_FMT_NONE && avctx->pix_fmt != ctx->pix_fmt) {
        av_log(avctx, AV_LOG_WARNING, "pix_fmt changed: %s -> %s\n",
               av_get_pix_fmt_name(avctx->pix_fmt), av_get_pix_fmt_name(ctx->pix_fmt));
        first_field = 1;
    }

    avctx->pix_fmt = ctx->pix_fmt;
    ret = ff_set_dimensions(avctx, ctx->width, ctx->height);
    if (ret < 0)
        return ret;

    if (first_field) {
        if ((ret = ff_thread_get_buffer(avctx, &frame, 0)) < 0)
            return ret;
        picture->pict_type = AV_PICTURE_TYPE_I;
        picture->key_frame = 1;
    }

    ctx->buf_size = buf_size - ctx->data_offset;
    ctx->buf = buf + ctx->data_offset;
    avctx->execute2(avctx, dnxhd_decode_row, picture, NULL, ctx->mb_height);

    if (first_field && picture->interlaced_frame) {
        buf      += ctx->cid_table->coding_unit_size;
        buf_size -= ctx->cid_table->coding_unit_size;
        first_field = 0;
        goto decode_coding_unit;
    }

    ret = 0;
    for (i = 0; i < avctx->thread_count; i++) {
        ret += ctx->rows[i].errors;
        ctx->rows[i].errors = 0;
    }

    if (ctx->act) {
        static int act_warned;
        int format = ctx->rows[0].format;
        for (i = 1; i < avctx->thread_count; i++) {
            if (ctx->rows[i].format != format &&
                ctx->rows[i].format != -1 /* not run */) {
                format = 2;
                break;
            }
        }
        switch (format) {
        case -1:
        case 2:
            if (!act_warned) {
                act_warned = 1;
                av_log(ctx->avctx, AV_LOG_ERROR,
                       "Unsupported: variable ACT flag.\n");
            }
            break;
        case 0:
            ctx->pix_fmt = ctx->bit_depth==10
                         ? AV_PIX_FMT_GBRP10 : AV_PIX_FMT_GBRP12;
            break;
        case 1:
            ctx->pix_fmt = ctx->bit_depth==10
                         ? AV_PIX_FMT_YUV444P10 : AV_PIX_FMT_YUV444P12;
            break;
        }
    }
    avctx->pix_fmt = ctx->pix_fmt;
    if (ret) {
        av_log(ctx->avctx, AV_LOG_ERROR, "%d lines with errors\n", ret);
        return AVERROR_INVALIDDATA;
    }

    *got_frame = 1;
    return avpkt->size;
}

Project: FFmpeg
File: dnxhddec.c

static int dnxhd_decode_row(AVCodecContext *avctx, void *data,
                            int rownb, int threadnb)
{
    const DNXHDContext *ctx = avctx->priv_data;
    uint32_t offset = ctx->mb_scan_index[rownb];
    RowContext *row = ctx->rows + threadnb;
    int x;

    row->last_dc[0] =
    row->last_dc[1] =
    row->last_dc[2] = 1 << (ctx->bit_depth + 2); // for levels +2^(bitdepth-1)
    init_get_bits(&row->gb, ctx->buf + offset, (ctx->buf_size - offset) << 3);
    for (x = 0; x < ctx->mb_width; x++) {
        //START_TIMER;
        int ret = dnxhd_decode_macroblock(ctx, row, data, x, rownb);
        if (ret < 0) {
            row->errors++;
            return ret;
        }
        //STOP_TIMER("decode macroblock");
    }

    return 0;
}