Commit message: avcodec/dnxhddec: check and propagate function return value

Similar to CVE-2013-0868, here return value check for 'init_vlc' is needed.
crafted DNxHD data can cause unspecified impact.

Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: James Almer <jamrial@gmail.com>
CVE: CVE-2021-38114
CWE: CWE-125
Vulnerability description: libavcodec/dnxhddec.c in FFmpeg 4.4 does not check the return value of the init_vlc function, a similar issue to CVE-2013-0868.

Project: FFmpeg
File: dnxhddec.c

static av_cold int dnxhd_decode_init(AVCodecContext *avctx)
{
    DNXHDContext *ctx = avctx->priv_data;

    ctx->avctx = avctx;
    ctx->cid = -1;
    if (avctx->colorspace == AVCOL_SPC_UNSPECIFIED) {
        avctx->colorspace = AVCOL_SPC_BT709;
    }

    avctx->coded_width  = FFALIGN(avctx->width,  16);
    avctx->coded_height = FFALIGN(avctx->height, 16);

    ctx->rows = av_mallocz_array(avctx->thread_count, sizeof(RowContext));
    if (!ctx->rows)
        return AVERROR(ENOMEM);

    return 0;
}

Project: FFmpeg
File: dnxhddec.c

static av_always_inline int dnxhd_decode_dct_block(const DNXHDContext *ctx,
                                                   RowContext *row,
                                                   int n,
                                                   int index_bits,
                                                   int level_bias,
                                                   int level_shift,
                                                   int dc_shift)
{
    int i, j, index1, index2, len, flags;
    int level, component, sign;
    const int *scale;
    const uint8_t *weight_matrix;
    const uint8_t *ac_info = ctx->cid_table->ac_info;
    int16_t *block = row->blocks[n];
    const int eob_index     = ctx->cid_table->eob_index;
    int ret = 0;
    OPEN_READER(bs, &row->gb);

    ctx->bdsp.clear_block(block);

    if (!ctx->is_444) {
        if (n & 2) {
            component     = 1 + (n & 1);
            scale = row->chroma_scale;
            weight_matrix = ctx->cid_table->chroma_weight;
        } else {
            component     = 0;
            scale = row->luma_scale;
            weight_matrix = ctx->cid_table->luma_weight;
        }
    } else {
        component = (n >> 1) % 3;
        if (component) {
            scale = row->chroma_scale;
            weight_matrix = ctx->cid_table->chroma_weight;
        } else {
            scale = row->luma_scale;
            weight_matrix = ctx->cid_table->luma_weight;
        }
    }

    UPDATE_CACHE(bs, &row->gb);
    GET_VLC(len, bs, &row->gb, ctx->dc_vlc.table, DNXHD_DC_VLC_BITS, 1);
    if (len < 0) {
        ret = len;
        goto error;
    }
    if (len) {
        level = GET_CACHE(bs, &row->gb);
        LAST_SKIP_BITS(bs, &row->gb, len);
        sign  = ~level >> 31;
        level = (NEG_USR32(sign ^ level, len) ^ sign) - sign;
        row->last_dc[component] += level * (1 << dc_shift);
    }
    block[0] = row->last_dc[component];

    i = 0;

    UPDATE_CACHE(bs, &row->gb);
    GET_VLC(index1, bs, &row->gb, ctx->ac_vlc.table,
            DNXHD_VLC_BITS, 2);

    while (index1 != eob_index) {
        level = ac_info[2*index1+0];
        flags = ac_info[2*index1+1];

        sign = SHOW_SBITS(bs, &row->gb, 1);
        SKIP_BITS(bs, &row->gb, 1);

        if (flags & 1) {
            level += SHOW_UBITS(bs, &row->gb, index_bits) << 7;
            SKIP_BITS(bs, &row->gb, index_bits);
        }

        if (flags & 2) {
            UPDATE_CACHE(bs, &row->gb);
            GET_VLC(index2, bs, &row->gb, ctx->run_vlc.table,
                    DNXHD_VLC_BITS, 2);
            i += ctx->cid_table->run[index2];
        }

        if (++i > 63) {
            av_log(ctx->avctx, AV_LOG_ERROR, "ac tex damaged %d, %d\n", n, i);
            ret = -1;
            break;
        }

        j     = ctx->scantable.permutated[i];
        level *= scale[i];
        level += scale[i] >> 1;
        if (level_bias < 32 || weight_matrix[i] != level_bias)
            level += level_bias; // 1<<(level_shift-1)
        level >>= level_shift;

        block[j] = (level ^ sign) - sign;

        UPDATE_CACHE(bs, &row->gb);
        GET_VLC(index1, bs, &row->gb, ctx->ac_vlc.table,
                DNXHD_VLC_BITS, 2);
    }
error:
    CLOSE_READER(bs, &row->gb);
    return ret;
}

Project: FFmpeg
File: dnxhddec.c

static int dnxhd_init_vlc(DNXHDContext *ctx, uint32_t cid, int bitdepth)
{
    int ret;
    if (cid != ctx->cid) {
        const CIDEntry *cid_table = ff_dnxhd_get_cid_table(cid);

        if (!cid_table) {
            av_log(ctx->avctx, AV_LOG_ERROR, "unsupported cid %"PRIu32"\n", cid);
            return AVERROR(ENOSYS);
        }
        if (cid_table->bit_depth != bitdepth &&
            cid_table->bit_depth != DNXHD_VARIABLE) {
            av_log(ctx->avctx, AV_LOG_ERROR, "bit depth mismatches %d %d\n",
                   cid_table->bit_depth, bitdepth);
            return AVERROR_INVALIDDATA;
        }
        ctx->cid_table = cid_table;
        av_log(ctx->avctx, AV_LOG_VERBOSE, "Profile cid %"PRIu32".\n", cid);

        ff_free_vlc(&ctx->ac_vlc);
        ff_free_vlc(&ctx->dc_vlc);
        ff_free_vlc(&ctx->run_vlc);

        if ((ret = init_vlc(&ctx->ac_vlc, DNXHD_VLC_BITS, 257,
                 ctx->cid_table->ac_bits, 1, 1,
                 ctx->cid_table->ac_codes, 2, 2, 0)) < 0)
            goto out;
        if ((ret = init_vlc(&ctx->dc_vlc, DNXHD_DC_VLC_BITS, bitdepth > 8 ? 14 : 12,
                 ctx->cid_table->dc_bits, 1, 1,
                 ctx->cid_table->dc_codes, 1, 1, 0)) < 0)
            goto out;
        if ((ret = init_vlc(&ctx->run_vlc, DNXHD_VLC_BITS, 62,
                 ctx->cid_table->run_bits, 1, 1,
                 ctx->cid_table->run_codes, 2, 2, 0)) < 0)
            goto out;

        ctx->cid = cid;
    }
    ret = 0;
out:
    if (ret < 0)
        av_log(ctx->avctx, AV_LOG_ERROR, "init_vlc failed\n");
    return ret;
}

Project: FFmpeg
File: dnxhddec.c

static int dnxhd_decode_header(DNXHDContext *ctx, AVFrame *frame,
                               const uint8_t *buf, int buf_size,
                               int first_field)
{
    int i, cid, ret;
    int old_bit_depth = ctx->bit_depth, bitdepth;
    uint64_t header_prefix;
    if (buf_size < 0x280) {
        av_log(ctx->avctx, AV_LOG_ERROR,
               "buffer too small (%d < 640).\n", buf_size);
        return AVERROR_INVALIDDATA;
    }

    header_prefix = ff_dnxhd_parse_header_prefix(buf);
    if (header_prefix == 0) {
        av_log(ctx->avctx, AV_LOG_ERROR,
               "unknown header 0x%02X 0x%02X 0x%02X 0x%02X 0x%02X\n",
               buf[0], buf[1], buf[2], buf[3], buf[4]);
        return AVERROR_INVALIDDATA;
    }
    if (buf[5] & 2) { /* interlaced */
        ctx->cur_field = buf[5] & 1;
        frame->interlaced_frame = 1;
        frame->top_field_first  = first_field ^ ctx->cur_field;
        av_log(ctx->avctx, AV_LOG_DEBUG,
               "interlaced %d, cur field %d\n", buf[5] & 3, ctx->cur_field);
    } else {
        ctx->cur_field = 0;
    }
    ctx->mbaff = (buf[0x6] >> 5) & 1;
    ctx->alpha = buf[0x7] & 1;
    ctx->lla   = (buf[0x7] >> 1) & 1;
    if (ctx->alpha)
        avpriv_request_sample(ctx->avctx, "alpha");

    ctx->height = AV_RB16(buf + 0x18);
    ctx->width  = AV_RB16(buf + 0x1a);

    switch(buf[0x21] >> 5) {
    case 1: bitdepth = 8; break;
    case 2: bitdepth = 10; break;
    case 3: bitdepth = 12; break;
    default:
        av_log(ctx->avctx, AV_LOG_ERROR,
               "Unknown bitdepth indicator (%d)\n", buf[0x21] >> 5);
        return AVERROR_INVALIDDATA;
    }

    cid = AV_RB32(buf + 0x28);

    ctx->avctx->profile = dnxhd_get_profile(cid);

    if ((ret = dnxhd_init_vlc(ctx, cid, bitdepth)) < 0)
        return ret;
    if (ctx->mbaff && ctx->cid_table->cid != 1260)
        av_log(ctx->avctx, AV_LOG_WARNING,
               "Adaptive MB interlace flag in an unsupported profile.\n");

    switch ((buf[0x2C] >> 1) & 3) {
    case 0: frame->colorspace = AVCOL_SPC_BT709;       break;
    case 1: frame->colorspace = AVCOL_SPC_BT2020_NCL;  break;
    case 2: frame->colorspace = AVCOL_SPC_BT2020_CL;   break;
    case 3: frame->colorspace = AVCOL_SPC_UNSPECIFIED; break;
    }

    ctx->act = buf[0x2C] & 1;
    if (ctx->act && ctx->cid_table->cid != 1256 && ctx->cid_table->cid != 1270)
        av_log(ctx->avctx, AV_LOG_WARNING,
               "Adaptive color transform in an unsupported profile.\n");

    ctx->is_444 = (buf[0x2C] >> 6) & 1;
    if (ctx->is_444) {
        if (bitdepth == 8) {
            avpriv_request_sample(ctx->avctx, "4:4:4 8 bits");
            return AVERROR_INVALIDDATA;
        } else if (bitdepth == 10) {
            ctx->decode_dct_block = dnxhd_decode_dct_block_10_444;
            ctx->pix_fmt = ctx->act ? AV_PIX_FMT_YUV444P10
                                    : AV_PIX_FMT_GBRP10;
        } else {
            ctx->decode_dct_block = dnxhd_decode_dct_block_12_444;
            ctx->pix_fmt = ctx->act ? AV_PIX_FMT_YUV444P12
                                    : AV_PIX_FMT_GBRP12;
        }
    } else if (bitdepth == 12) {
        ctx->decode_dct_block = dnxhd_decode_dct_block_12;
        ctx->pix_fmt = AV_PIX_FMT_YUV422P12;
    } else if (bitdepth == 10) {
        if (ctx->avctx->profile == FF_PROFILE_DNXHR_HQX)
            ctx->decode_dct_block = dnxhd_decode_dct_block_10_444;
        else
            ctx->decode_dct_block = dnxhd_decode_dct_block_10;
        ctx->pix_fmt = AV_PIX_FMT_YUV422P10;
    } else {
        ctx->decode_dct_block = dnxhd_decode_dct_block_8;
        ctx->pix_fmt = AV_PIX_FMT_YUV422P;
    }

    ctx->avctx->bits_per_raw_sample = ctx->bit_depth = bitdepth;
    if (ctx->bit_depth != old_bit_depth) {
        ff_blockdsp_init(&ctx->bdsp, ctx->avctx);
        ff_idctdsp_init(&ctx->idsp, ctx->avctx);
        ff_init_scantable(ctx->idsp.idct_permutation, &ctx->scantable,
                          ff_zigzag_direct);
    }

    // make sure profile size constraints are respected
    // DNx100 allows 1920->1440 and 1280->960 subsampling
    if (ctx->width != ctx->cid_table->width &&
        ctx->cid_table->width != DNXHD_VARIABLE) {
        av_reduce(&ctx->avctx->sample_aspect_ratio.num,
                  &ctx->avctx->sample_aspect_ratio.den,
                  ctx->width, ctx->cid_table->width, 255);
        ctx->width = ctx->cid_table->width;
    }

    if (buf_size < ctx->cid_table->coding_unit_size) {
        av_log(ctx->avctx, AV_LOG_ERROR, "incorrect frame size (%d < %u).\n",
               buf_size, ctx->cid_table->coding_unit_size);
        return AVERROR_INVALIDDATA;
    }

    ctx->mb_width  = (ctx->width + 15)>> 4;
    ctx->mb_height = AV_RB16(buf + 0x16c);

    if ((ctx->height + 15) >> 4 == ctx->mb_height && frame->interlaced_frame)
        ctx->height <<= 1;

    av_log(ctx->avctx, AV_LOG_VERBOSE, "%dx%d, 4:%s %d bits, MBAFF=%d ACT=%d\n",
           ctx->width, ctx->height, ctx->is_444 ? "4:4" : "2:2",
           ctx->bit_depth, ctx->mbaff, ctx->act);

    // Newer format supports variable mb_scan_index sizes
    if (ctx->mb_height > 68 && ff_dnxhd_check_header_prefix_hr(header_prefix)) {
        ctx->data_offset = 0x170 + (ctx->mb_height << 2);
    } else {
        if (ctx->mb_height > 68) {
            av_log(ctx->avctx, AV_LOG_ERROR,
                   "mb height too big: %d\n", ctx->mb_height);
            return AVERROR_INVALIDDATA;
        }
        ctx->data_offset = 0x280;
    }
    if ((ctx->mb_height << frame->interlaced_frame) > (ctx->height + 15) >> 4) {
        av_log(ctx->avctx, AV_LOG_ERROR,
                "mb height too big: %d\n", ctx->mb_height);
        return AVERROR_INVALIDDATA;
    }

    if (buf_size < ctx->data_offset) {
        av_log(ctx->avctx, AV_LOG_ERROR,
               "buffer too small (%d < %d).\n", buf_size, ctx->data_offset);
        return AVERROR_INVALIDDATA;
    }

    if (ctx->mb_height > FF_ARRAY_ELEMS(ctx->mb_scan_index)) {
        av_log(ctx->avctx, AV_LOG_ERROR,
               "mb_height too big (%d > %"SIZE_SPECIFIER").\n", ctx->mb_height, FF_ARRAY_ELEMS(ctx->mb_scan_index));
        return AVERROR_INVALIDDATA;
    }

    for (i = 0; i < ctx->mb_height; i++) {
        ctx->mb_scan_index[i] = AV_RB32(buf + 0x170 + (i << 2));
        ff_dlog(ctx->avctx, "mb scan index %d, pos %d: %"PRIu32"\n",
                i, 0x170 + (i << 2), ctx->mb_scan_index[i]);
        if (buf_size - ctx->data_offset < ctx->mb_scan_index[i]) {
            av_log(ctx->avctx, AV_LOG_ERROR,
                   "invalid mb scan index (%"PRIu32" vs %u).\n",
                   ctx->mb_scan_index[i], buf_size - ctx->data_offset);
            return AVERROR_INVALIDDATA;
        }
    }

    return 0;
}

Project: FFmpeg
File: dnxhddec.c

static int dnxhd_decode_row(AVCodecContext *avctx, void *data,
                            int rownb, int threadnb)
{
    const DNXHDContext *ctx = avctx->priv_data;
    uint32_t offset = ctx->mb_scan_index[rownb];
    RowContext *row = ctx->rows + threadnb;
    int x, ret;

    row->last_dc[0] =
    row->last_dc[1] =
    row->last_dc[2] = 1 << (ctx->bit_depth + 2); // for levels +2^(bitdepth-1)
    ret = init_get_bits8(&row->gb, ctx->buf + offset, ctx->buf_size - offset);
    if (ret < 0) {
        row->errors++;
        return ret;
    }
    for (x = 0; x < ctx->mb_width; x++) {
        int ret = dnxhd_decode_macroblock(ctx, row, data, x, rownb);
        if (ret < 0) {
            row->errors++;
            return ret;
        }
    }

    return 0;
}