Commit message: avcodec/apedec: Fix integer overflow

Fixes: out of array access
Fixes: PoC.ape and others

Found-by: Bingchang, Liu@VARAS of IIE
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit ba4beaf6149f7241c8bd85fe853318c2f6837ad0)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
CVE: None
Vulnerability description: None

Project: FFmpeg
File: apedec.c

static void entropy_decode_stereo_3900(APEContext *ctx, int blockstodecode)
{
    int32_t *decoded0 = ctx->decoded[0];
    int32_t *decoded1 = ctx->decoded[1];
    int blocks = blockstodecode;

    while (blockstodecode--)
        *decoded0++ = ape_decode_value_3900(ctx, &ctx->riceY);
    range_dec_normalize(ctx);
    // because of some implementation peculiarities we need to backpedal here
    ctx->ptr -= 1;
    range_start_decoding(ctx);
    while (blocks--)
        *decoded1++ = ape_decode_value_3900(ctx, &ctx->riceX);
}

Project: FFmpeg
File: apedec.c

static int ape_decode_frame(AVCodecContext *avctx, void *data,
                            int *got_frame_ptr, AVPacket *avpkt)
{
    AVFrame *frame     = data;
    const uint8_t *buf = avpkt->data;
    APEContext *s = avctx->priv_data;
    uint8_t *sample8;
    int16_t *sample16;
    int32_t *sample24;
    int i, ch, ret;
    int blockstodecode;
    uint64_t decoded_buffer_size;

    /* this should never be negative, but bad things will happen if it is, so
       check it just to make sure. */
    av_assert0(s->samples >= 0);

    if(!s->samples){
        uint32_t nblocks, offset;
        int buf_size;

        if (!avpkt->size) {
            *got_frame_ptr = 0;
            return 0;
        }
        if (avpkt->size < 8) {
            av_log(avctx, AV_LOG_ERROR, "Packet is too small\n");
            return AVERROR_INVALIDDATA;
        }
        buf_size = avpkt->size & ~3;
        if (buf_size != avpkt->size) {
            av_log(avctx, AV_LOG_WARNING, "packet size is not a multiple of 4. "
                   "extra bytes at the end will be skipped.\n");
        }
        if (s->fileversion < 3950) // previous versions overread two bytes
            buf_size += 2;
        av_fast_padded_malloc(&s->data, &s->data_size, buf_size);
        if (!s->data)
            return AVERROR(ENOMEM);
        s->bdsp.bswap_buf((uint32_t *) s->data, (const uint32_t *) buf,
                          buf_size >> 2);
        memset(s->data + (buf_size & ~3), 0, buf_size & 3);
        s->ptr = s->data;
        s->data_end = s->data + buf_size;

        nblocks = bytestream_get_be32(&s->ptr);
        offset  = bytestream_get_be32(&s->ptr);
        if (s->fileversion >= 3900) {
            if (offset > 3) {
                av_log(avctx, AV_LOG_ERROR, "Incorrect offset passed\n");
                s->data = NULL;
                return AVERROR_INVALIDDATA;
            }
            if (s->data_end - s->ptr < offset) {
                av_log(avctx, AV_LOG_ERROR, "Packet is too small\n");
                return AVERROR_INVALIDDATA;
            }
            s->ptr += offset;
        } else {
            if ((ret = init_get_bits8(&s->gb, s->ptr, s->data_end - s->ptr)) < 0)
                return ret;
            if (s->fileversion > 3800)
                skip_bits_long(&s->gb, offset * 8);
            else
                skip_bits_long(&s->gb, offset);
        }

        if (!nblocks || nblocks > INT_MAX / 2 / sizeof(*s->decoded_buffer) - 8) {
            av_log(avctx, AV_LOG_ERROR, "Invalid sample count: %"PRIu32".\n",
                   nblocks);
            return AVERROR_INVALIDDATA;
        }

        /* Initialize the frame decoder */
        if (init_frame_decoder(s) < 0) {
            av_log(avctx, AV_LOG_ERROR, "Error reading frame header\n");
            return AVERROR_INVALIDDATA;
        }
        s->samples = nblocks;
    }

    if (!s->data) {
        *got_frame_ptr = 0;
        return avpkt->size;
    }

    blockstodecode = FFMIN(s->blocks_per_loop, s->samples);
    // for old files coefficients were not interleaved,
    // so we need to decode all of them at once
    if (s->fileversion < 3930)
        blockstodecode = s->samples;

    /* reallocate decoded sample buffer if needed */
    decoded_buffer_size = 2LL * FFALIGN(blockstodecode, 8) * sizeof(*s->decoded_buffer);
    av_assert0(decoded_buffer_size <= INT_MAX);
    av_fast_malloc(&s->decoded_buffer, &s->decoded_size, decoded_buffer_size);
    if (!s->decoded_buffer)
        return AVERROR(ENOMEM);
    memset(s->decoded_buffer, 0, s->decoded_size);
    s->decoded[0] = s->decoded_buffer;
    s->decoded[1] = s->decoded_buffer + FFALIGN(blockstodecode, 8);

    /* get output buffer */
    frame->nb_samples = blockstodecode;
    if ((ret = ff_get_buffer(avctx, frame, 0)) < 0)
        return ret;

    s->error=0;

    if ((s->channels == 1) || (s->frameflags & APE_FRAMECODE_PSEUDO_STEREO))
        ape_unpack_mono(s, blockstodecode);
    else
        ape_unpack_stereo(s, blockstodecode);
    emms_c();

    if (s->error) {
        s->samples=0;
        av_log(avctx, AV_LOG_ERROR, "Error decoding frame\n");
        return AVERROR_INVALIDDATA;
    }

    switch (s->bps) {
    case 8:
        for (ch = 0; ch < s->channels; ch++) {
            sample8 = (uint8_t *)frame->data[ch];
            for (i = 0; i < blockstodecode; i++)
                *sample8++ = (s->decoded[ch][i] + 0x80) & 0xff;
        }
        break;
    case 16:
        for (ch = 0; ch < s->channels; ch++) {
            sample16 = (int16_t *)frame->data[ch];
            for (i = 0; i < blockstodecode; i++)
                *sample16++ = s->decoded[ch][i];
        }
        break;
    case 24:
        for (ch = 0; ch < s->channels; ch++) {
            sample24 = (int32_t *)frame->data[ch];
            for (i = 0; i < blockstodecode; i++)
                *sample24++ = s->decoded[ch][i] << 8;
        }
        break;
    }

    s->samples -= blockstodecode;

    *got_frame_ptr = 1;

    return !s->samples ? avpkt->size : 0;
}

Project: FFmpeg
File: apedec.c

static av_cold int ape_decode_init(AVCodecContext *avctx)
{
    APEContext *s = avctx->priv_data;
    int i;

    if (avctx->extradata_size != 6) {
        av_log(avctx, AV_LOG_ERROR, "Incorrect extradata\n");
        return AVERROR(EINVAL);
    }
    if (avctx->channels > 2) {
        av_log(avctx, AV_LOG_ERROR, "Only mono and stereo is supported\n");
        return AVERROR(EINVAL);
    }
    s->bps = avctx->bits_per_coded_sample;
    switch (s->bps) {
    case 8:
        avctx->sample_fmt = AV_SAMPLE_FMT_U8P;
        break;
    case 16:
        avctx->sample_fmt = AV_SAMPLE_FMT_S16P;
        break;
    case 24:
        avctx->sample_fmt = AV_SAMPLE_FMT_S32P;
        break;
    default:
        avpriv_request_sample(avctx,
                              "%d bits per coded sample", s->bps);
        return AVERROR_PATCHWELCOME;
    }
    s->avctx             = avctx;
    s->channels          = avctx->channels;
    s->fileversion       = AV_RL16(avctx->extradata);
    s->compression_level = AV_RL16(avctx->extradata + 2);
    s->flags             = AV_RL16(avctx->extradata + 4);

    av_log(avctx, AV_LOG_DEBUG, "Compression Level: %d - Flags: %d\n",
           s->compression_level, s->flags);
    if (s->compression_level % 1000 || s->compression_level > COMPRESSION_LEVEL_INSANE ||
        !s->compression_level ||
        (s->fileversion < 3930 && s->compression_level == COMPRESSION_LEVEL_INSANE)) {
        av_log(avctx, AV_LOG_ERROR, "Incorrect compression level %d\n",
               s->compression_level);
        return AVERROR_INVALIDDATA;
    }
    s->fset = s->compression_level / 1000 - 1;
    for (i = 0; i < APE_FILTER_LEVELS; i++) {
        if (!ape_filter_orders[s->fset][i])
            break;
        FF_ALLOC_OR_GOTO(avctx, s->filterbuf[i],
                         (ape_filter_orders[s->fset][i] * 3 + HISTORY_SIZE) * 4,
                         filter_alloc_fail);
    }

    if (s->fileversion < 3860) {
        s->entropy_decode_mono   = entropy_decode_mono_0000;
        s->entropy_decode_stereo = entropy_decode_stereo_0000;
    } else if (s->fileversion < 3900) {
        s->entropy_decode_mono   = entropy_decode_mono_3860;
        s->entropy_decode_stereo = entropy_decode_stereo_3860;
    } else if (s->fileversion < 3930) {
        s->entropy_decode_mono   = entropy_decode_mono_3900;
        s->entropy_decode_stereo = entropy_decode_stereo_3900;
    } else if (s->fileversion < 3990) {
        s->entropy_decode_mono   = entropy_decode_mono_3900;
        s->entropy_decode_stereo = entropy_decode_stereo_3930;
    } else {
        s->entropy_decode_mono   = entropy_decode_mono_3990;
        s->entropy_decode_stereo = entropy_decode_stereo_3990;
    }

    if (s->fileversion < 3930) {
        s->predictor_decode_mono   = predictor_decode_mono_3800;
        s->predictor_decode_stereo = predictor_decode_stereo_3800;
    } else if (s->fileversion < 3950) {
        s->predictor_decode_mono   = predictor_decode_mono_3930;
        s->predictor_decode_stereo = predictor_decode_stereo_3930;
    } else {
        s->predictor_decode_mono   = predictor_decode_mono_3950;
        s->predictor_decode_stereo = predictor_decode_stereo_3950;
    }

    ff_bswapdsp_init(&s->bdsp);
    ff_llauddsp_init(&s->adsp);
    avctx->channel_layout = (avctx->channels==2) ? AV_CH_LAYOUT_STEREO : AV_CH_LAYOUT_MONO;

    return 0;
filter_alloc_fail:
    ape_decode_close(avctx);
    return AVERROR(ENOMEM);
}

Project: FFmpeg
File: apedec.c

static inline int ape_decode_value_3990(APEContext *ctx, APERice *rice)
{
    unsigned int x, overflow;
    int base, pivot;

    pivot = rice->ksum >> 5;
    if (pivot == 0)
        pivot = 1;

    overflow = range_get_symbol(ctx, counts_3980, counts_diff_3980);

    if (overflow == (MODEL_ELEMENTS - 1)) {
        overflow  = range_decode_bits(ctx, 16) << 16;
        overflow |= range_decode_bits(ctx, 16);
    }

    if (pivot < 0x10000) {
        base = range_decode_culfreq(ctx, pivot);
        range_decode_update(ctx, 1, base);
    } else {
        int base_hi = pivot, base_lo;
        int bbits = 0;

        while (base_hi & ~0xFFFF) {
            base_hi >>= 1;
            bbits++;
        }
        base_hi = range_decode_culfreq(ctx, base_hi + 1);
        range_decode_update(ctx, 1, base_hi);
        base_lo = range_decode_culfreq(ctx, 1 << bbits);
        range_decode_update(ctx, 1, base_lo);

        base = (base_hi << bbits) + base_lo;
    }

    x = base + overflow * pivot;

    update_rice(rice, x);

    /* Convert to signed */
    if (x & 1)
        return (x >> 1) + 1;
    else
        return -(x >> 1);
}

Project: FFmpeg
File: apedec.c

static inline int ape_decode_value_3860(APEContext *ctx, GetBitContext *gb,
                                        APERice *rice)
{
    unsigned int x, overflow;

    overflow = get_unary(gb, 1, get_bits_left(gb));

    if (ctx->fileversion > 3880) {
        while (overflow >= 16) {
            overflow -= 16;
            rice->k  += 4;
        }
    }

    if (!rice->k)
        x = overflow;
    else if(rice->k <= MIN_CACHE_BITS) {
        x = (overflow << rice->k) + get_bits(gb, rice->k);
    } else {
        av_log(ctx->avctx, AV_LOG_ERROR, "Too many bits: %d\n", rice->k);
        return AVERROR_INVALIDDATA;
    }
    rice->ksum += x - (rice->ksum + 8 >> 4);
    if (rice->ksum < (rice->k ? 1 << (rice->k + 4) : 0))
        rice->k--;
    else if (rice->ksum >= (1 << (rice->k + 5)) && rice->k < 24)
        rice->k++;

    /* Convert to signed */
    if (x & 1)
        return (x >> 1) + 1;
    else
        return -(x >> 1);
}

Project: FFmpeg
File: apedec.c

static void long_filter_high_3800(int32_t *buffer, int order, int shift,
                                  int32_t *coeffs, int32_t *delay, int length)
{
    int i, j;
    int32_t dotprod, sign;

    if (order >= length)
        return;

    memset(coeffs, 0, order * sizeof(*coeffs));
    for (i = 0; i < order; i++)
        delay[i] = buffer[i];
    for (i = order; i < length; i++) {
        dotprod = 0;
        sign = APESIGN(buffer[i]);
        for (j = 0; j < order; j++) {
            dotprod += delay[j] * coeffs[j];
            coeffs[j] += ((delay[j] >> 31) | 1) * sign;
        }
        buffer[i] -= dotprod >> shift;
        for (j = 0; j < order - 1; j++)
            delay[j] = delay[j + 1];
        delay[order - 1] = buffer[i];
    }
}

Project: FFmpeg
File: apedec.c

static void predictor_decode_mono_3800(APEContext *ctx, int count)
{
    APEPredictor *p = &ctx->predictor;
    int32_t *decoded0 = ctx->decoded[0];
    int32_t coeffs[256], delay[256];
    int start = 4, shift = 10;

    if (ctx->compression_level == COMPRESSION_LEVEL_HIGH) {
        start = 16;
        long_filter_high_3800(decoded0, 16, 9, coeffs, delay, count);
    } else if (ctx->compression_level == COMPRESSION_LEVEL_EXTRA_HIGH) {
        int order = 128, shift2 = 11;

        if (ctx->fileversion >= 3830) {
            order <<= 1;
            shift++;
            shift2++;
            long_filter_ehigh_3830(decoded0 + order, count - order);
        }
        start = order;
        long_filter_high_3800(decoded0, order, shift2, coeffs, delay, count);
    }

    while (count--) {
        if (ctx->compression_level == COMPRESSION_LEVEL_FAST) {
            *decoded0 = filter_fast_3320(p, *decoded0, 0, YDELAYA);
            decoded0++;
        } else {
            *decoded0 = filter_3800(p, *decoded0, 0, YDELAYA, YDELAYB,
                                    start, shift);
            decoded0++;
        }

        /* Combined */
        p->buf++;
        p->sample_pos++;

        /* Have we filled the history buffer? */
        if (p->buf == p->historybuffer + HISTORY_SIZE) {
            memmove(p->historybuffer, p->buf,
                    PREDICTOR_SIZE * sizeof(*p->historybuffer));
            p->buf = p->historybuffer;
        }
    }
}

Project: FFmpeg
File: apedec.c

static void do_apply_filter(APEContext *ctx, int version, APEFilter *f,
                            int32_t *data, int count, int order, int fracbits)
{
    int res;
    int absres;

    while (count--) {
        /* round fixedpoint scalar product */
        res = ctx->adsp.scalarproduct_and_madd_int16(f->coeffs,
                                                     f->delay - order,
                                                     f->adaptcoeffs - order,
                                                     order, APESIGN(*data));
        res = (res + (1 << (fracbits - 1))) >> fracbits;
        res += *data;
        *data++ = res;

        /* Update the output history */
        *f->delay++ = av_clip_int16(res);

        if (version < 3980) {
            /* Version ??? to < 3.98 files (untested) */
            f->adaptcoeffs[0]  = (res == 0) ? 0 : ((res >> 28) & 8) - 4;
            f->adaptcoeffs[-4] >>= 1;
            f->adaptcoeffs[-8] >>= 1;
        } else {
            /* Version 3.98 and later files */

            /* Update the adaption coefficients */
            absres = FFABS(res);
            if (absres)
                *f->adaptcoeffs = ((res & (-1<<31)) ^ (-1<<30)) >>
                                  (25 + (absres <= f->avg*3) + (absres <= f->avg*4/3));
            else
                *f->adaptcoeffs = 0;

            f->avg += (absres - f->avg) / 16;

            f->adaptcoeffs[-1] >>= 1;
            f->adaptcoeffs[-2] >>= 1;
            f->adaptcoeffs[-8] >>= 1;
        }

        f->adaptcoeffs++;

        /* Have we filled the history buffer? */
        if (f->delay == f->historybuffer + HISTORY_SIZE + (order * 2)) {
            memmove(f->historybuffer, f->delay - (order * 2),
                    (order * 2) * sizeof(*f->historybuffer));
            f->delay = f->historybuffer + order * 2;
            f->adaptcoeffs = f->historybuffer + order;
        }
    }
}

Project: FFmpeg
File: apedec.c

static void decode_array_0000(APEContext *ctx, GetBitContext *gb,
                              int32_t *out, APERice *rice, int blockstodecode)
{
    int i;
    int ksummax, ksummin;

    rice->ksum = 0;
    for (i = 0; i < FFMIN(blockstodecode, 5); i++) {
        out[i] = get_rice_ook(&ctx->gb, 10);
        rice->ksum += out[i];
    }
    rice->k = av_log2(rice->ksum / 10) + 1;
    if (rice->k >= 24)
        return;
    for (; i < FFMIN(blockstodecode, 64); i++) {
        out[i] = get_rice_ook(&ctx->gb, rice->k);
        rice->ksum += out[i];
        rice->k = av_log2(rice->ksum / ((i + 1) * 2)) + 1;
        if (rice->k >= 24)
            return;
    }
    ksummax = 1 << rice->k + 7;
    ksummin = rice->k ? (1 << rice->k + 6) : 0;
    for (; i < blockstodecode; i++) {
        out[i] = get_rice_ook(&ctx->gb, rice->k);
        rice->ksum += out[i] - out[i - 64];
        while (rice->ksum < ksummin) {
            rice->k--;
            ksummin = rice->k ? ksummin >> 1 : 0;
            ksummax >>= 1;
        }
        while (rice->ksum >= ksummax) {
            rice->k++;
            if (rice->k > 24)
                return;
            ksummax <<= 1;
            ksummin = ksummin ? ksummin << 1 : 128;
        }
    }

    for (i = 0; i < blockstodecode; i++) {
        if (out[i] & 1)
            out[i] = (out[i] >> 1) + 1;
        else
            out[i] = -(out[i] >> 1);
    }
}

Project: FFmpeg
File: apedec.c

static void predictor_decode_stereo_3800(APEContext *ctx, int count)
{
    APEPredictor *p = &ctx->predictor;
    int32_t *decoded0 = ctx->decoded[0];
    int32_t *decoded1 = ctx->decoded[1];
    int32_t coeffs[256], delay[256];
    int start = 4, shift = 10;

    if (ctx->compression_level == COMPRESSION_LEVEL_HIGH) {
        start = 16;
        long_filter_high_3800(decoded0, 16, 9, coeffs, delay, count);
        long_filter_high_3800(decoded1, 16, 9, coeffs, delay, count);
    } else if (ctx->compression_level == COMPRESSION_LEVEL_EXTRA_HIGH) {
        int order = 128, shift2 = 11;

        if (ctx->fileversion >= 3830) {
            order <<= 1;
            shift++;
            shift2++;
            long_filter_ehigh_3830(decoded0 + order, count - order);
            long_filter_ehigh_3830(decoded1 + order, count - order);
        }
        start = order;
        long_filter_high_3800(decoded0, order, shift2, coeffs, delay, count);
        long_filter_high_3800(decoded1, order, shift2, coeffs, delay, count);
    }

    while (count--) {
        int X = *decoded0, Y = *decoded1;
        if (ctx->compression_level == COMPRESSION_LEVEL_FAST) {
            *decoded0 = filter_fast_3320(p, Y, 0, YDELAYA);
            decoded0++;
            *decoded1 = filter_fast_3320(p, X, 1, XDELAYA);
            decoded1++;
        } else {
            *decoded0 = filter_3800(p, Y, 0, YDELAYA, YDELAYB,
                                    start, shift);
            decoded0++;
            *decoded1 = filter_3800(p, X, 1, XDELAYA, XDELAYB,
                                    start, shift);
            decoded1++;
        }

        /* Combined */
        p->buf++;
        p->sample_pos++;

        /* Have we filled the history buffer? */
        if (p->buf == p->historybuffer + HISTORY_SIZE) {
            memmove(p->historybuffer, p->buf,
                    PREDICTOR_SIZE * sizeof(*p->historybuffer));
            p->buf = p->historybuffer;
        }
    }
}

Project: FFmpeg
File: apedec.c

static inline int ape_decode_value_3900(APEContext *ctx, APERice *rice)
{
    unsigned int x, overflow;
    int tmpk;

    overflow = range_get_symbol(ctx, counts_3970, counts_diff_3970);

    if (overflow == (MODEL_ELEMENTS - 1)) {
        tmpk = range_decode_bits(ctx, 5);
        overflow = 0;
    } else
        tmpk = (rice->k < 1) ? 0 : rice->k - 1;

    if (tmpk <= 16 || ctx->fileversion < 3910) {
        if (tmpk > 23) {
            av_log(ctx->avctx, AV_LOG_ERROR, "Too many bits: %d\n", tmpk);
            return AVERROR_INVALIDDATA;
        }
        x = range_decode_bits(ctx, tmpk);
    } else if (tmpk <= 31) {
        x = range_decode_bits(ctx, 16);
        x |= (range_decode_bits(ctx, tmpk - 16) << 16);
    } else {
        av_log(ctx->avctx, AV_LOG_ERROR, "Too many bits: %d\n", tmpk);
        return AVERROR_INVALIDDATA;
    }
    x += overflow << tmpk;

    update_rice(rice, x);

    /* Convert to signed */
    if (x & 1)
        return (x >> 1) + 1;
    else
        return -(x >> 1);
}