Commit message: avcodec/jpeg2000dec: prevent out of array accesses in pixel addressing

Fixes Ticket2921

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
CVE: CVE-2013-7024
CWE: CWE-119
Vulnerability description: The jpeg2000_decode_tile function in libavcodec/jpeg2000dec.c in FFmpeg before 2.1 does not consider the component number in certain calculations, which allows remote attackers to cause a denial of service (out-of-bounds array access) or possibly have unspecified other impact via crafted JPEG2000 data.

Project: FFmpeg
File: jpeg2000dec.c

static void decode_refpass(Jpeg2000T1Context *t1, int width, int height,
                           int bpno)
{
    int phalf, nhalf;
    int y0, x, y;

    phalf = 1 << (bpno - 1);
    nhalf = -phalf;

    for (y0 = 0; y0 < height; y0 += 4)
        for (x = 0; x < width; x++)
            for (y = y0; y < height && y < y0 + 4; y++)
                if ((t1->flags[y + 1][x + 1] & (JPEG2000_T1_SIG | JPEG2000_T1_VIS)) == JPEG2000_T1_SIG) {
                    int ctxno = ff_jpeg2000_getrefctxno(t1->flags[y + 1][x + 1]);
                    int r     = ff_mqc_decode(&t1->mqc,
                                              t1->mqc.cx_states + ctxno)
                                ? phalf : nhalf;
                    t1->data[y][x]          += t1->data[y][x] < 0 ? -r : r;
                    t1->flags[y + 1][x + 1] |= JPEG2000_T1_REF;
                }
}

Project: FFmpeg
File: jpeg2000dec.c

static void decode_sigpass(Jpeg2000T1Context *t1, int width, int height,
                           int bpno, int bandno, int bpass_csty_symbol,
                           int vert_causal_ctx_csty_symbol)
{
    int mask = 3 << (bpno - 1), y0, x, y;

    for (y0 = 0; y0 < height; y0 += 4)
        for (x = 0; x < width; x++)
            for (y = y0; y < height && y < y0 + 4; y++) {
                if ((t1->flags[y+1][x+1] & JPEG2000_T1_SIG_NB)
                && !(t1->flags[y+1][x+1] & (JPEG2000_T1_SIG | JPEG2000_T1_VIS))) {
                    int flags_mask = -1;
                    if (vert_causal_ctx_csty_symbol && y == y0 + 3)
                        flags_mask &= ~(JPEG2000_T1_SIG_S | JPEG2000_T1_SIG_SW | JPEG2000_T1_SIG_SE);
                    if (ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ff_jpeg2000_getsigctxno(t1->flags[y+1][x+1] & flags_mask, bandno))) {
                        int xorbit, ctxno = ff_jpeg2000_getsgnctxno(t1->flags[y+1][x+1], &xorbit);
                        if (bpass_csty_symbol)
                             t1->data[y][x] = ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ctxno) ? -mask : mask;
                        else
                             t1->data[y][x] = (ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ctxno) ^ xorbit) ?
                                               -mask : mask;

                        ff_jpeg2000_set_significance(t1, x, y,
                                                     t1->data[y][x] < 0);
                    }
                    t1->flags[y + 1][x + 1] |= JPEG2000_T1_VIS;
                }
            }
}

Project: FFmpeg
File: jpeg2000dec.c

static int jpeg2000_decode_packet(Jpeg2000DecoderContext *s,
                                  Jpeg2000CodingStyle *codsty,
                                  Jpeg2000ResLevel *rlevel, int precno,
                                  int layno, uint8_t *expn, int numgbits)
{
    int bandno, cblkno, ret, nb_code_blocks;

    if (!(ret = get_bits(s, 1))) {
        jpeg2000_flush(s);
        return 0;
    } else if (ret < 0)
        return ret;

    for (bandno = 0; bandno < rlevel->nbands; bandno++) {
        Jpeg2000Band *band = rlevel->band + bandno;
        Jpeg2000Prec *prec = band->prec + precno;

        if (band->coord[0][0] == band->coord[0][1] ||
            band->coord[1][0] == band->coord[1][1])
            continue;
        nb_code_blocks =  prec->nb_codeblocks_height *
                          prec->nb_codeblocks_width;
        for (cblkno = 0; cblkno < nb_code_blocks; cblkno++) {
            Jpeg2000Cblk *cblk = prec->cblk + cblkno;
            int incl, newpasses, llen;

            if (cblk->npasses)
                incl = get_bits(s, 1);
            else
                incl = tag_tree_decode(s, prec->cblkincl + cblkno, layno + 1) == layno;
            if (!incl)
                continue;
            else if (incl < 0)
                return incl;

            if (!cblk->npasses) {
                int v = expn[bandno] + numgbits - 1 -
                        tag_tree_decode(s, prec->zerobits + cblkno, 100);
                if (v < 0) {
                    av_log(s->avctx, AV_LOG_ERROR,
                           "nonzerobits %d invalid\n", v);
                    return AVERROR_INVALIDDATA;
                }
                cblk->nonzerobits = v;
            }
            if ((newpasses = getnpasses(s)) < 0)
                return newpasses;
            if ((llen = getlblockinc(s)) < 0)
                return llen;
            cblk->lblock += llen;
            if ((ret = get_bits(s, av_log2(newpasses) + cblk->lblock)) < 0)
                return ret;
            if (ret > sizeof(cblk->data)) {
                avpriv_request_sample(s->avctx,
                                      "Block with lengthinc greater than %zu",
                                      sizeof(cblk->data));
                return AVERROR_PATCHWELCOME;
            }
            cblk->lengthinc = ret;
            cblk->npasses  += newpasses;
        }
    }
    jpeg2000_flush(s);

    if (codsty->csty & JPEG2000_CSTY_EPH) {
        if (bytestream2_peek_be16(&s->g) == JPEG2000_EPH)
            bytestream2_skip(&s->g, 2);
        else
            av_log(s->avctx, AV_LOG_ERROR, "EPH marker not found.\n");
    }

    for (bandno = 0; bandno < rlevel->nbands; bandno++) {
        Jpeg2000Band *band = rlevel->band + bandno;
        Jpeg2000Prec *prec = band->prec + precno;

        nb_code_blocks = prec->nb_codeblocks_height * prec->nb_codeblocks_width;
        for (cblkno = 0; cblkno < nb_code_blocks; cblkno++) {
            Jpeg2000Cblk *cblk = prec->cblk + cblkno;
            if (   bytestream2_get_bytes_left(&s->g) < cblk->lengthinc
                || sizeof(cblk->data) < cblk->length + cblk->lengthinc + 2
            ) {
                av_log(s->avctx, AV_LOG_ERROR,
                       "Block length %d or lengthinc %d is too large\n",
                       cblk->length, cblk->lengthinc);
                return AVERROR_INVALIDDATA;
            }

            bytestream2_get_bufferu(&s->g, cblk->data + cblk->length, cblk->lengthinc);
            cblk->length   += cblk->lengthinc;
            cblk->lengthinc = 0;
        }
    }
    return 0;
}

Project: FFmpeg

static int jpeg2000_read_main_headers(Jpeg2000DecoderContext *s)
{
    Jpeg2000CodingStyle *codsty = s->codsty;
    Jpeg2000QuantStyle *qntsty  = s->qntsty;
    uint8_t *properties         = s->properties;

    for (;;) {
        int len, ret = 0;
        uint16_t marker;
        int oldpos;

        if (bytestream2_get_bytes_left(&s->g) < 2) {
            av_log(s->avctx, AV_LOG_ERROR, "Missing EOC\n");
            break;
        }

        marker = bytestream2_get_be16u(&s->g);
        oldpos = bytestream2_tell(&s->g);

        if (marker == JPEG2000_SOD) {
            Jpeg2000Tile *tile;
            Jpeg2000TilePart *tp;

            if (!s->tile) {
                av_log(s->avctx, AV_LOG_ERROR, "Missing SIZ\n");
                return AVERROR_INVALIDDATA;
            }
            if (s->curtileno < 0) {
                av_log(s->avctx, AV_LOG_ERROR, "Missing SOT\n");
                return AVERROR_INVALIDDATA;
            }

            tile = s->tile + s->curtileno;
            tp = tile->tile_part + tile->tp_idx;
            if (tp->tp_end < s->g.buffer) {
                av_log(s->avctx, AV_LOG_ERROR, "Invalid tpend\n");
                return AVERROR_INVALIDDATA;
            }
            bytestream2_init(&tp->tpg, s->g.buffer, tp->tp_end - s->g.buffer);
            bytestream2_skip(&s->g, tp->tp_end - s->g.buffer);

            continue;
        }
        if (marker == JPEG2000_EOC)
            break;

        len = bytestream2_get_be16(&s->g);
        if (len < 2 || bytestream2_get_bytes_left(&s->g) < len - 2)
            return AVERROR_INVALIDDATA;

        switch (marker) {
        case JPEG2000_SIZ:
            ret = get_siz(s);
            if (!s->tile)
                s->numXtiles = s->numYtiles = 0;
            break;
        case JPEG2000_COC:
            ret = get_coc(s, codsty, properties);
            break;
        case JPEG2000_COD:
            ret = get_cod(s, codsty, properties);
            break;
        case JPEG2000_QCC:
            ret = get_qcc(s, len, qntsty, properties);
            break;
        case JPEG2000_QCD:
            ret = get_qcd(s, len, qntsty, properties);
            break;
        case JPEG2000_SOT:
            if (!(ret = get_sot(s, len))) {
                av_assert1(s->curtileno >= 0);
                codsty = s->tile[s->curtileno].codsty;
                qntsty = s->tile[s->curtileno].qntsty;
                properties = s->tile[s->curtileno].properties;
            }
            break;
        case JPEG2000_COM:
            bytestream2_skip(&s->g, len - 2);
            break;
        case JPEG2000_TLM:
            ret = get_tlm(s, len);
            break;
        default:
            av_log(s->avctx, AV_LOG_ERROR,
                   "unsupported marker 0x%.4X at pos 0x%X\n",
                   marker, bytestream2_tell(&s->g) - 4);
            bytestream2_skip(&s->g, len - 2);
            break;
        }
        if (bytestream2_tell(&s->g) - oldpos != len || ret) {
            av_log(s->avctx, AV_LOG_ERROR,
                   "error during processing marker segment %.4x\n", marker);
            return ret ? ret : -1;
        }
    }
    return 0;
}