Commit message: Avoid division by zero in opj_pi_next_rpcl, opj_pi_next_pcrl and opj_pi_next_cprl (#938)

Fixes issues with id:000026,sig:08,src:002419,op:int32,pos:60,val:+32 and
id:000019,sig:08,src:001098,op:flip1,pos:49
CVE: CVE-2016-10506
CWE: CWE-369
Vulnerability description: Division-by-zero vulnerabilities in the functions opj_pi_next_cprl, opj_pi_next_pcrl, and opj_pi_next_rpcl in pi.c in OpenJPEG before 2.2.0 allow remote attackers to cause a denial of service (application crash) via crafted j2k files.

Project: openjpeg
File: pi.c

static OPJ_BOOL opj_pi_next_pcrl(opj_pi_iterator_t * pi)
{
    opj_pi_comp_t *comp = NULL;
    opj_pi_resolution_t *res = NULL;
    OPJ_UINT32 index = 0;

    if (!pi->first) {
        comp = &pi->comps[pi->compno];
        goto LABEL_SKIP;
    } else {
        OPJ_UINT32 compno, resno;
        pi->first = 0;
        pi->dx = 0;
        pi->dy = 0;
        for (compno = 0; compno < pi->numcomps; compno++) {
            comp = &pi->comps[compno];
            for (resno = 0; resno < comp->numresolutions; resno++) {
                OPJ_UINT32 dx, dy;
                res = &comp->resolutions[resno];
                dx = comp->dx * (1u << (res->pdx + comp->numresolutions - 1 - resno));
                dy = comp->dy * (1u << (res->pdy + comp->numresolutions - 1 - resno));
                pi->dx = !pi->dx ? dx : opj_uint_min(pi->dx, dx);
                pi->dy = !pi->dy ? dy : opj_uint_min(pi->dy, dy);
            }
        }
    }
    if (!pi->tp_on) {
        pi->poc.ty0 = pi->ty0;
        pi->poc.tx0 = pi->tx0;
        pi->poc.ty1 = pi->ty1;
        pi->poc.tx1 = pi->tx1;
    }
    for (pi->y = pi->poc.ty0; pi->y < pi->poc.ty1;
            pi->y += (OPJ_INT32)(pi->dy - (OPJ_UINT32)(pi->y % (OPJ_INT32)pi->dy))) {
        for (pi->x = pi->poc.tx0; pi->x < pi->poc.tx1;
                pi->x += (OPJ_INT32)(pi->dx - (OPJ_UINT32)(pi->x % (OPJ_INT32)pi->dx))) {
            for (pi->compno = pi->poc.compno0; pi->compno < pi->poc.compno1; pi->compno++) {
                comp = &pi->comps[pi->compno];
                for (pi->resno = pi->poc.resno0;
                        pi->resno < opj_uint_min(pi->poc.resno1, comp->numresolutions); pi->resno++) {
                    OPJ_UINT32 levelno;
                    OPJ_INT32 trx0, try0;
                    OPJ_INT32 trx1, try1;
                    OPJ_UINT32 rpx, rpy;
                    OPJ_INT32 prci, prcj;
                    res = &comp->resolutions[pi->resno];
                    levelno = comp->numresolutions - 1 - pi->resno;
                    trx0 = opj_int_ceildiv(pi->tx0, (OPJ_INT32)(comp->dx << levelno));
                    try0 = opj_int_ceildiv(pi->ty0, (OPJ_INT32)(comp->dy << levelno));
                    trx1 = opj_int_ceildiv(pi->tx1, (OPJ_INT32)(comp->dx << levelno));
                     try1 = opj_int_ceildiv(pi->ty1, (OPJ_INT32)(comp->dy << levelno));
                     rpx = res->pdx + levelno;
                     rpy = res->pdy + levelno;

                    /* To avoid divisions by zero / undefined behaviour on shift */
                    /* in below tests */
                    /* Relates to id:000019,sig:08,src:001098,op:flip1,pos:49 */
                    /* of https://github.com/uclouvain/openjpeg/issues/938 */
                    if (rpx >= 31 || ((comp->dx << rpx) >> rpx) != comp->dx ||
                            rpy >= 31 || ((comp->dy << rpy) >> rpy) != comp->dy) {
                        continue;
                    }

                    /* See ISO-15441. B.12.1.4 Position-component-resolution level-layer progression */
                     if (!((pi->y % (OPJ_INT32)(comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&
                             ((try0 << levelno) % (1 << rpy))))) {
                         continue;
                    }
                    if (!((pi->x % (OPJ_INT32)(comp->dx << rpx) == 0) || ((pi->x == pi->tx0) &&
                            ((trx0 << levelno) % (1 << rpx))))) {
                        continue;
                    }

                    if ((res->pw == 0) || (res->ph == 0)) {
                        continue;
                    }

                    if ((trx0 == trx1) || (try0 == try1)) {
                        continue;
                    }

                    prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x,
                                                (OPJ_INT32)(comp->dx << levelno)), (OPJ_INT32)res->pdx)
                           - opj_int_floordivpow2(trx0, (OPJ_INT32)res->pdx);
                    prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y,
                                                (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
                           - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
                    pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
                    for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
                        index = pi->layno * pi->step_l + pi->resno * pi->step_r + pi->compno *
                                pi->step_c + pi->precno * pi->step_p;
                        if (!pi->include[index]) {
                            pi->include[index] = 1;
                            return OPJ_TRUE;
                        }
LABEL_SKIP:
                        ;
                    }
                }
            }
        }
    }

    return OPJ_FALSE;
}

Project: openjpeg
File: pi.c

static OPJ_BOOL opj_pi_next_rpcl(opj_pi_iterator_t * pi)
{
    opj_pi_comp_t *comp = NULL;
    opj_pi_resolution_t *res = NULL;
    OPJ_UINT32 index = 0;

    if (!pi->first) {
        goto LABEL_SKIP;
    } else {
        OPJ_UINT32 compno, resno;
        pi->first = 0;
        pi->dx = 0;
        pi->dy = 0;
        for (compno = 0; compno < pi->numcomps; compno++) {
            comp = &pi->comps[compno];
            for (resno = 0; resno < comp->numresolutions; resno++) {
                OPJ_UINT32 dx, dy;
                res = &comp->resolutions[resno];
                dx = comp->dx * (1u << (res->pdx + comp->numresolutions - 1 - resno));
                dy = comp->dy * (1u << (res->pdy + comp->numresolutions - 1 - resno));
                pi->dx = !pi->dx ? dx : opj_uint_min(pi->dx, dx);
                pi->dy = !pi->dy ? dy : opj_uint_min(pi->dy, dy);
            }
        }
    }
    if (!pi->tp_on) {
        pi->poc.ty0 = pi->ty0;
        pi->poc.tx0 = pi->tx0;
        pi->poc.ty1 = pi->ty1;
        pi->poc.tx1 = pi->tx1;
    }
    for (pi->resno = pi->poc.resno0; pi->resno < pi->poc.resno1; pi->resno++) {
        for (pi->y = pi->poc.ty0; pi->y < pi->poc.ty1;
                pi->y += (OPJ_INT32)(pi->dy - (OPJ_UINT32)(pi->y % (OPJ_INT32)pi->dy))) {
            for (pi->x = pi->poc.tx0; pi->x < pi->poc.tx1;
                    pi->x += (OPJ_INT32)(pi->dx - (OPJ_UINT32)(pi->x % (OPJ_INT32)pi->dx))) {
                for (pi->compno = pi->poc.compno0; pi->compno < pi->poc.compno1; pi->compno++) {
                    OPJ_UINT32 levelno;
                    OPJ_INT32 trx0, try0;
                    OPJ_INT32  trx1, try1;
                    OPJ_UINT32  rpx, rpy;
                    OPJ_INT32  prci, prcj;
                    comp = &pi->comps[pi->compno];
                    if (pi->resno >= comp->numresolutions) {
                        continue;
                    }
                    res = &comp->resolutions[pi->resno];
                    levelno = comp->numresolutions - 1 - pi->resno;
                    trx0 = opj_int_ceildiv(pi->tx0, (OPJ_INT32)(comp->dx << levelno));
                    try0 = opj_int_ceildiv(pi->ty0, (OPJ_INT32)(comp->dy << levelno));
                    trx1 = opj_int_ceildiv(pi->tx1, (OPJ_INT32)(comp->dx << levelno));
                     try1 = opj_int_ceildiv(pi->ty1, (OPJ_INT32)(comp->dy << levelno));
                     rpx = res->pdx + levelno;
                     rpy = res->pdy + levelno;

                    /* To avoid divisions by zero / undefined behaviour on shift */
                    /* in below tests */
                    /* Fixes reading id:000026,sig:08,src:002419,op:int32,pos:60,val:+32 */
                    /* of https://github.com/uclouvain/openjpeg/issues/938 */
                    if (rpx >= 31 || ((comp->dx << rpx) >> rpx) != comp->dx ||
                            rpy >= 31 || ((comp->dy << rpy) >> rpy) != comp->dy) {
                        continue;
                    }

                    /* See ISO-15441. B.12.1.3 Resolution level-position-component-layer progression */
                     if (!((pi->y % (OPJ_INT32)(comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&
                             ((try0 << levelno) % (1 << rpy))))) {
                         continue;
                    }
                    if (!((pi->x % (OPJ_INT32)(comp->dx << rpx) == 0) || ((pi->x == pi->tx0) &&
                            ((trx0 << levelno) % (1 << rpx))))) {
                        continue;
                    }

                    if ((res->pw == 0) || (res->ph == 0)) {
                        continue;
                    }

                    if ((trx0 == trx1) || (try0 == try1)) {
                        continue;
                    }

                    prci = opj_int_floordivpow2(opj_int_ceildiv(pi->x,
                                                (OPJ_INT32)(comp->dx << levelno)), (OPJ_INT32)res->pdx)
                           - opj_int_floordivpow2(trx0, (OPJ_INT32)res->pdx);
                    prcj = opj_int_floordivpow2(opj_int_ceildiv(pi->y,
                                                (OPJ_INT32)(comp->dy << levelno)), (OPJ_INT32)res->pdy)
                           - opj_int_floordivpow2(try0, (OPJ_INT32)res->pdy);
                    pi->precno = (OPJ_UINT32)(prci + prcj * (OPJ_INT32)res->pw);
                    for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
                        index = pi->layno * pi->step_l + pi->resno * pi->step_r + pi->compno *
                                pi->step_c + pi->precno * pi->step_p;
                        if (!pi->include[index]) {
                            pi->include[index] = 1;
                            return OPJ_TRUE;
                        }
LABEL_SKIP:
                        ;
                    }
                }
            }
        }
    }

    return OPJ_FALSE;
}