Commit message: [OPENJP2] change the way to compute *p_tx0, *p_tx1, *p_ty0, *p_ty1 in function
opj_get_encoding_parameters

Signed-off-by: Young_X <YangX92@hotmail.com>
CVE: CVE-2018-20847
CWE: CWE-190
Vulnerability description: An improper computation of p_tx0, p_tx1, p_ty0 and p_ty1 in the function opj_get_encoding_parameters in openjp2/pi.c in OpenJPEG through 2.3.0 can lead to an integer overflow.

Project: openjpeg
File: pi.c

static OPJ_BOOL opj_pi_next_rlcp(opj_pi_iterator_t * pi)
{
    opj_pi_comp_t *comp = NULL;
    opj_pi_resolution_t *res = NULL;
    OPJ_UINT32 index = 0;

    if (!pi->first) {
        comp = &pi->comps[pi->compno];
        res = &comp->resolutions[pi->resno];
        goto LABEL_SKIP;
    } else {
        pi->first = 0;
    }

    for (pi->resno = pi->poc.resno0; pi->resno < pi->poc.resno1; pi->resno++) {
        for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
            for (pi->compno = pi->poc.compno0; pi->compno < pi->poc.compno1; pi->compno++) {
                comp = &pi->comps[pi->compno];
                if (pi->resno >= comp->numresolutions) {
                    continue;
                }
                res = &comp->resolutions[pi->resno];
                if (!pi->tp_on) {
                    pi->poc.precno1 = res->pw * res->ph;
                }
                for (pi->precno = pi->poc.precno0; pi->precno < pi->poc.precno1; pi->precno++) {
                    index = pi->layno * pi->step_l + pi->resno * pi->step_r + pi->compno *
                            pi->step_c + pi->precno * pi->step_p;
                    if (index >= pi->include_size) {
                        opj_pi_emit_error(pi, "Invalid access to pi->include");
                        return OPJ_FALSE;
                    }
                    if (!pi->include[index]) {
                        pi->include[index] = 1;
                        return OPJ_TRUE;
                    }
LABEL_SKIP:
                    ;
                }
            }
        }
    }

    return OPJ_FALSE;
}

Project: openjpeg
File: pi.c

static void opj_pi_update_decode_poc(opj_pi_iterator_t * p_pi,
                                     opj_tcp_t * p_tcp,
                                     OPJ_UINT32 p_max_precision,
                                     OPJ_UINT32 p_max_res)
{
    /* loop*/
    OPJ_UINT32 pino;

    /* encoding prameters to set*/
    OPJ_UINT32 l_bound;

    opj_pi_iterator_t * l_current_pi = 00;
    opj_poc_t* l_current_poc = 0;

    OPJ_ARG_NOT_USED(p_max_res);

    /* preconditions in debug*/
    assert(p_pi != 00);
    assert(p_tcp != 00);

    /* initializations*/
    l_bound = p_tcp->numpocs + 1;
    l_current_pi = p_pi;
    l_current_poc = p_tcp->pocs;

    for (pino = 0; pino < l_bound; ++pino) {
        l_current_pi->poc.prg = l_current_poc->prg; /* Progression Order #0 */
        l_current_pi->first = 1;

        l_current_pi->poc.resno0 =
            l_current_poc->resno0; /* Resolution Level Index #0 (Start) */
        l_current_pi->poc.compno0 =
            l_current_poc->compno0; /* Component Index #0 (Start) */
        l_current_pi->poc.layno0 = 0;
        l_current_pi->poc.precno0 = 0;
        l_current_pi->poc.resno1 =
            l_current_poc->resno1; /* Resolution Level Index #0 (End) */
        l_current_pi->poc.compno1 =
            l_current_poc->compno1; /* Component Index #0 (End) */
        l_current_pi->poc.layno1 = opj_uint_min(l_current_poc->layno1,
                                                p_tcp->numlayers); /* Layer Index #0 (End) */
        l_current_pi->poc.precno1 = p_max_precision;
        ++l_current_pi;
        ++l_current_poc;
    }
}