Commit message: [MJ2] Avoid index out of bounds access to pi->include[]

Signed-off-by: Young_X <YangX92@hotmail.com>
CVE: CVE-2018-20846
CWE: CWE-20
Vulnerability description: Out-of-bounds accesses in the functions pi_next_lrcp, pi_next_rlcp, pi_next_rpcl, pi_next_pcrl, pi_next_rpcl, and pi_next_cprl in openmj2/pi.c in OpenJPEG through 2.3.0 allow remote attackers to cause a denial of service (application crash).

Project: openjpeg
File: pi.c

opj_pi_iterator_t *pi_create_decode(opj_image_t *image, opj_cp_t *cp,
                                    int tileno)
{
    int p, q;
    int compno, resno, pino;
    opj_pi_iterator_t *pi = NULL;
    opj_tcp_t *tcp = NULL;
    opj_tccp_t *tccp = NULL;

    tcp = &cp->tcps[tileno];

    pi = (opj_pi_iterator_t*) opj_calloc((tcp->numpocs + 1),
                                         sizeof(opj_pi_iterator_t));
    if (!pi) {
        /* TODO: throw an error */
        return NULL;
    }

    for (pino = 0; pino < tcp->numpocs + 1; pino++) {   /* change */
        int maxres = 0;
        int maxprec = 0;
        p = tileno % cp->tw;
        q = tileno / cp->tw;

        pi[pino].tx0 = int_max(cp->tx0 + p * cp->tdx, image->x0);
        pi[pino].ty0 = int_max(cp->ty0 + q * cp->tdy, image->y0);
        pi[pino].tx1 = int_min(cp->tx0 + (p + 1) * cp->tdx, image->x1);
        pi[pino].ty1 = int_min(cp->ty0 + (q + 1) * cp->tdy, image->y1);
        pi[pino].numcomps = image->numcomps;

        pi[pino].comps = (opj_pi_comp_t*) opj_calloc(image->numcomps,
                         sizeof(opj_pi_comp_t));
        if (!pi[pino].comps) {
            /* TODO: throw an error */
            pi_destroy(pi, cp, tileno);
            return NULL;
        }

        for (compno = 0; compno < pi->numcomps; compno++) {
            int tcx0, tcy0, tcx1, tcy1;
            opj_pi_comp_t *comp = &pi[pino].comps[compno];
            tccp = &tcp->tccps[compno];
            comp->dx = image->comps[compno].dx;
            comp->dy = image->comps[compno].dy;
            comp->numresolutions = tccp->numresolutions;

            comp->resolutions = (opj_pi_resolution_t*) opj_calloc(comp->numresolutions,
                                sizeof(opj_pi_resolution_t));
            if (!comp->resolutions) {
                /* TODO: throw an error */
                pi_destroy(pi, cp, tileno);
                return NULL;
            }

            tcx0 = int_ceildiv(pi->tx0, comp->dx);
            tcy0 = int_ceildiv(pi->ty0, comp->dy);
            tcx1 = int_ceildiv(pi->tx1, comp->dx);
            tcy1 = int_ceildiv(pi->ty1, comp->dy);
            if (comp->numresolutions > maxres) {
                maxres = comp->numresolutions;
            }

            for (resno = 0; resno < comp->numresolutions; resno++) {
                int levelno;
                int rx0, ry0, rx1, ry1;
                int px0, py0, px1, py1;
                opj_pi_resolution_t *res = &comp->resolutions[resno];
                if (tccp->csty & J2K_CCP_CSTY_PRT) {
                    res->pdx = tccp->prcw[resno];
                    res->pdy = tccp->prch[resno];
                } else {
                    res->pdx = 15;
                    res->pdy = 15;
                }
                levelno = comp->numresolutions - 1 - resno;
                rx0 = int_ceildivpow2(tcx0, levelno);
                ry0 = int_ceildivpow2(tcy0, levelno);
                rx1 = int_ceildivpow2(tcx1, levelno);
                ry1 = int_ceildivpow2(tcy1, levelno);
                px0 = int_floordivpow2(rx0, res->pdx) << res->pdx;
                py0 = int_floordivpow2(ry0, res->pdy) << res->pdy;
                px1 = int_ceildivpow2(rx1, res->pdx) << res->pdx;
                py1 = int_ceildivpow2(ry1, res->pdy) << res->pdy;
                res->pw = (rx0 == rx1) ? 0 : ((px1 - px0) >> res->pdx);
                res->ph = (ry0 == ry1) ? 0 : ((py1 - py0) >> res->pdy);

                if (res->pw * res->ph > maxprec) {
                    maxprec = res->pw * res->ph;
                }

            }
        }

        tccp = &tcp->tccps[0];
        pi[pino].step_p = 1;
        pi[pino].step_c = maxprec * pi[pino].step_p;
        pi[pino].step_r = image->numcomps * pi[pino].step_c;
        pi[pino].step_l = maxres * pi[pino].step_r;

        if (pino == 0) {
            pi[pino].include = (short int*) opj_calloc(image->numcomps * maxres *
                               tcp->numlayers * maxprec, sizeof(short int));
            if (!pi[pino].include) {
                /* TODO: throw an error */
                pi_destroy(pi, cp, tileno);
                return NULL;
            }
        } else {
            pi[pino].include = pi[pino - 1].include;
        }

        if (tcp->POC == 0) {
            pi[pino].first = 1;
            pi[pino].poc.resno0 = 0;
            pi[pino].poc.compno0 = 0;
            pi[pino].poc.layno1 = tcp->numlayers;
            pi[pino].poc.resno1 = maxres;
            pi[pino].poc.compno1 = image->numcomps;
            pi[pino].poc.prg = tcp->prg;
        } else {
            pi[pino].first = 1;
            pi[pino].poc.resno0 = tcp->pocs[pino].resno0;
            pi[pino].poc.compno0 = tcp->pocs[pino].compno0;
            pi[pino].poc.layno1 = tcp->pocs[pino].layno1;
            pi[pino].poc.resno1 = tcp->pocs[pino].resno1;
            pi[pino].poc.compno1 = tcp->pocs[pino].compno1;
            pi[pino].poc.prg = tcp->pocs[pino].prg;
        }
        pi[pino].poc.layno0  = 0;
        pi[pino].poc.precno0 = 0;
        pi[pino].poc.precno1 = maxprec;

    }

    return pi;
}

Project: openjpeg
File: pi.c

opj_bool pi_create_encode(opj_pi_iterator_t *pi, opj_cp_t *cp, int tileno,
                          int pino, int tpnum, int tppos, J2K_T2_MODE t2_mode, int cur_totnum_tp)
{
    char prog[4];
    int i;
    int incr_top = 1, resetX = 0;
    opj_tcp_t *tcps = &cp->tcps[tileno];
    opj_poc_t *tcp = &tcps->pocs[pino];

    pi[pino].first = 1;
    pi[pino].poc.prg = tcp->prg;

    switch (tcp->prg) {
    case CPRL:
        strncpy(prog, "CPRL", 4);
        break;
    case LRCP:
        strncpy(prog, "LRCP", 4);
        break;
    case PCRL:
        strncpy(prog, "PCRL", 4);
        break;
    case RLCP:
        strncpy(prog, "RLCP", 4);
        break;
    case RPCL:
        strncpy(prog, "RPCL", 4);
        break;
    case PROG_UNKNOWN:
        return OPJ_TRUE;
    }

    if (!(cp->tp_on && ((!cp->cinema && (t2_mode == FINAL_PASS)) || cp->cinema))) {
        pi[pino].poc.resno0 = tcp->resS;
        pi[pino].poc.resno1 = tcp->resE;
        pi[pino].poc.compno0 = tcp->compS;
        pi[pino].poc.compno1 = tcp->compE;
        pi[pino].poc.layno0 = tcp->layS;
        pi[pino].poc.layno1 = tcp->layE;
        pi[pino].poc.precno0 = tcp->prcS;
        pi[pino].poc.precno1 = tcp->prcE;
        pi[pino].poc.tx0 = tcp->txS;
        pi[pino].poc.ty0 = tcp->tyS;
        pi[pino].poc.tx1 = tcp->txE;
        pi[pino].poc.ty1 = tcp->tyE;
    } else {
        if (tpnum < cur_totnum_tp) {
            for (i = 3; i >= 0; i--) {
                switch (prog[i]) {
                case 'C':
                    if (i > tppos) {
                        pi[pino].poc.compno0 = tcp->compS;
                        pi[pino].poc.compno1 = tcp->compE;
                    } else {
                        if (tpnum == 0) {
                            tcp->comp_t = tcp->compS;
                            pi[pino].poc.compno0 = tcp->comp_t;
                            pi[pino].poc.compno1 = tcp->comp_t + 1;
                            tcp->comp_t += 1;
                        } else {
                            if (incr_top == 1) {
                                if (tcp->comp_t == tcp->compE) {
                                    tcp->comp_t = tcp->compS;
                                    pi[pino].poc.compno0 = tcp->comp_t;
                                    pi[pino].poc.compno1 = tcp->comp_t + 1;
                                    tcp->comp_t += 1;
                                    incr_top = 1;
                                } else {
                                    pi[pino].poc.compno0 = tcp->comp_t;
                                    pi[pino].poc.compno1 = tcp->comp_t + 1;
                                    tcp->comp_t += 1;
                                    incr_top = 0;
                                }
                            } else {
                                pi[pino].poc.compno0 = tcp->comp_t - 1;
                                pi[pino].poc.compno1 = tcp->comp_t;
                            }
                        }
                    }
                    break;

                case 'R':
                    if (i > tppos) {
                        pi[pino].poc.resno0 = tcp->resS;
                        pi[pino].poc.resno1 = tcp->resE;
                    } else {
                        if (tpnum == 0) {
                            tcp->res_t = tcp->resS;
                            pi[pino].poc.resno0 = tcp->res_t;
                            pi[pino].poc.resno1 = tcp->res_t + 1;
                            tcp->res_t += 1;
                        } else {
                            if (incr_top == 1) {
                                if (tcp->res_t == tcp->resE) {
                                    tcp->res_t = tcp->resS;
                                    pi[pino].poc.resno0 = tcp->res_t;
                                    pi[pino].poc.resno1 = tcp->res_t + 1;
                                    tcp->res_t += 1;
                                    incr_top = 1;
                                } else {
                                    pi[pino].poc.resno0 = tcp->res_t;
                                    pi[pino].poc.resno1 = tcp->res_t + 1;
                                    tcp->res_t += 1;
                                    incr_top = 0;
                                }
                            } else {
                                pi[pino].poc.resno0 = tcp->res_t - 1;
                                pi[pino].poc.resno1 = tcp->res_t;
                            }
                        }
                    }
                    break;

                case 'L':
                    if (i > tppos) {
                        pi[pino].poc.layno0 = tcp->layS;
                        pi[pino].poc.layno1 = tcp->layE;
                    } else {
                        if (tpnum == 0) {
                            tcp->lay_t = tcp->layS;
                            pi[pino].poc.layno0 = tcp->lay_t;
                            pi[pino].poc.layno1 = tcp->lay_t + 1;
                            tcp->lay_t += 1;
                        } else {
                            if (incr_top == 1) {
                                if (tcp->lay_t == tcp->layE) {
                                    tcp->lay_t = tcp->layS;
                                    pi[pino].poc.layno0 = tcp->lay_t;
                                    pi[pino].poc.layno1 = tcp->lay_t + 1;
                                    tcp->lay_t += 1;
                                    incr_top = 1;
                                } else {
                                    pi[pino].poc.layno0 = tcp->lay_t;
                                    pi[pino].poc.layno1 = tcp->lay_t + 1;
                                    tcp->lay_t += 1;
                                    incr_top = 0;
                                }
                            } else {
                                pi[pino].poc.layno0 = tcp->lay_t - 1;
                                pi[pino].poc.layno1 = tcp->lay_t;
                            }
                        }
                    }
                    break;

                case 'P':
                    switch (tcp->prg) {
                    case LRCP:
                    case RLCP:
                        if (i > tppos) {
                            pi[pino].poc.precno0 = tcp->prcS;
                            pi[pino].poc.precno1 = tcp->prcE;
                        } else {
                            if (tpnum == 0) {
                                tcp->prc_t = tcp->prcS;
                                pi[pino].poc.precno0 = tcp->prc_t;
                                pi[pino].poc.precno1 = tcp->prc_t + 1;
                                tcp->prc_t += 1;
                            } else {
                                if (incr_top == 1) {
                                    if (tcp->prc_t == tcp->prcE) {
                                        tcp->prc_t = tcp->prcS;
                                        pi[pino].poc.precno0 = tcp->prc_t;
                                        pi[pino].poc.precno1 = tcp->prc_t + 1;
                                        tcp->prc_t += 1;
                                        incr_top = 1;
                                    } else {
                                        pi[pino].poc.precno0 = tcp->prc_t;
                                        pi[pino].poc.precno1 = tcp->prc_t + 1;
                                        tcp->prc_t += 1;
                                        incr_top = 0;
                                    }
                                } else {
                                    pi[pino].poc.precno0 = tcp->prc_t - 1;
                                    pi[pino].poc.precno1 = tcp->prc_t;
                                }
                            }
                        }
                        break;
                    default:
                        if (i > tppos) {
                            pi[pino].poc.tx0 = tcp->txS;
                            pi[pino].poc.ty0 = tcp->tyS;
                            pi[pino].poc.tx1 = tcp->txE;
                            pi[pino].poc.ty1 = tcp->tyE;
                        } else {
                            if (tpnum == 0) {
                                tcp->tx0_t = tcp->txS;
                                tcp->ty0_t = tcp->tyS;
                                pi[pino].poc.tx0 = tcp->tx0_t;
                                pi[pino].poc.tx1 = tcp->tx0_t + tcp->dx - (tcp->tx0_t % tcp->dx);
                                pi[pino].poc.ty0 = tcp->ty0_t;
                                pi[pino].poc.ty1 = tcp->ty0_t + tcp->dy - (tcp->ty0_t % tcp->dy);
                                tcp->tx0_t = pi[pino].poc.tx1;
                                tcp->ty0_t = pi[pino].poc.ty1;
                            } else {
                                if (incr_top == 1) {
                                    if (tcp->tx0_t >= tcp->txE) {
                                        if (tcp->ty0_t >= tcp->tyE) {
                                            tcp->ty0_t = tcp->tyS;
                                            pi[pino].poc.ty0 = tcp->ty0_t;
                                            pi[pino].poc.ty1 = tcp->ty0_t + tcp->dy - (tcp->ty0_t % tcp->dy);
                                            tcp->ty0_t = pi[pino].poc.ty1;
                                            incr_top = 1;
                                            resetX = 1;
                                        } else {
                                            pi[pino].poc.ty0 = tcp->ty0_t;
                                            pi[pino].poc.ty1 = tcp->ty0_t + tcp->dy - (tcp->ty0_t % tcp->dy);
                                            tcp->ty0_t = pi[pino].poc.ty1;
                                            incr_top = 0;
                                            resetX = 1;
                                        }
                                        if (resetX == 1) {
                                            tcp->tx0_t = tcp->txS;
                                            pi[pino].poc.tx0 = tcp->tx0_t;
                                            pi[pino].poc.tx1 = tcp->tx0_t + tcp->dx - (tcp->tx0_t % tcp->dx);
                                            tcp->tx0_t = pi[pino].poc.tx1;
                                        }
                                    } else {
                                        pi[pino].poc.tx0 = tcp->tx0_t;
                                        pi[pino].poc.tx1 = tcp->tx0_t + tcp->dx - (tcp->tx0_t % tcp->dx);
                                        tcp->tx0_t = pi[pino].poc.tx1;
                                        pi[pino].poc.ty0 = tcp->ty0_t - tcp->dy - (tcp->ty0_t % tcp->dy);
                                        pi[pino].poc.ty1 = tcp->ty0_t ;
                                        incr_top = 0;
                                    }
                                } else {
                                    pi[pino].poc.tx0 = tcp->tx0_t - tcp->dx - (tcp->tx0_t % tcp->dx);
                                    pi[pino].poc.tx1 = tcp->tx0_t ;
                                    pi[pino].poc.ty0 = tcp->ty0_t - tcp->dy - (tcp->ty0_t % tcp->dy);
                                    pi[pino].poc.ty1 = tcp->ty0_t ;
                                }
                            }
                        }
                        break;
                    }
                    break;
                }
            }
        }
    }
    return OPJ_FALSE;
}

Project: openjpeg
File: pi.c

opj_pi_iterator_t *pi_initialise_encode(opj_image_t *image, opj_cp_t *cp,
                                        int tileno, J2K_T2_MODE t2_mode)
{
    int p, q, pino;
    int compno, resno;
    int maxres = 0;
    int maxprec = 0;
    opj_pi_iterator_t *pi = NULL;
    opj_tcp_t *tcp = NULL;
    opj_tccp_t *tccp = NULL;

    tcp = &cp->tcps[tileno];

    pi = (opj_pi_iterator_t*) opj_calloc((tcp->numpocs + 1),
                                         sizeof(opj_pi_iterator_t));
    if (!pi) {
        return NULL;
    }
    pi->tp_on = cp->tp_on;

    for (pino = 0; pino < tcp->numpocs + 1 ; pino ++) {
        p = tileno % cp->tw;
        q = tileno / cp->tw;

        pi[pino].tx0 = int_max(cp->tx0 + p * cp->tdx, image->x0);
        pi[pino].ty0 = int_max(cp->ty0 + q * cp->tdy, image->y0);
        pi[pino].tx1 = int_min(cp->tx0 + (p + 1) * cp->tdx, image->x1);
        pi[pino].ty1 = int_min(cp->ty0 + (q + 1) * cp->tdy, image->y1);
        pi[pino].numcomps = image->numcomps;

        pi[pino].comps = (opj_pi_comp_t*) opj_calloc(image->numcomps,
                         sizeof(opj_pi_comp_t));
        if (!pi[pino].comps) {
            pi_destroy(pi, cp, tileno);
            return NULL;
        }

        for (compno = 0; compno < pi[pino].numcomps; compno++) {
            int tcx0, tcy0, tcx1, tcy1;
            opj_pi_comp_t *comp = &pi[pino].comps[compno];
            tccp = &tcp->tccps[compno];
            comp->dx = image->comps[compno].dx;
            comp->dy = image->comps[compno].dy;
            comp->numresolutions = tccp->numresolutions;

            comp->resolutions = (opj_pi_resolution_t*) opj_malloc(comp->numresolutions *
                                sizeof(opj_pi_resolution_t));
            if (!comp->resolutions) {
                pi_destroy(pi, cp, tileno);
                return NULL;
            }

            tcx0 = int_ceildiv(pi[pino].tx0, comp->dx);
            tcy0 = int_ceildiv(pi[pino].ty0, comp->dy);
            tcx1 = int_ceildiv(pi[pino].tx1, comp->dx);
            tcy1 = int_ceildiv(pi[pino].ty1, comp->dy);
            if (comp->numresolutions > maxres) {
                maxres = comp->numresolutions;
            }

            for (resno = 0; resno < comp->numresolutions; resno++) {
                int levelno;
                int rx0, ry0, rx1, ry1;
                int px0, py0, px1, py1;
                opj_pi_resolution_t *res = &comp->resolutions[resno];
                if (tccp->csty & J2K_CCP_CSTY_PRT) {
                    res->pdx = tccp->prcw[resno];
                    res->pdy = tccp->prch[resno];
                } else {
                    res->pdx = 15;
                    res->pdy = 15;
                }
                levelno = comp->numresolutions - 1 - resno;
                rx0 = int_ceildivpow2(tcx0, levelno);
                ry0 = int_ceildivpow2(tcy0, levelno);
                rx1 = int_ceildivpow2(tcx1, levelno);
                ry1 = int_ceildivpow2(tcy1, levelno);
                px0 = int_floordivpow2(rx0, res->pdx) << res->pdx;
                py0 = int_floordivpow2(ry0, res->pdy) << res->pdy;
                px1 = int_ceildivpow2(rx1, res->pdx) << res->pdx;
                py1 = int_ceildivpow2(ry1, res->pdy) << res->pdy;
                res->pw = (rx0 == rx1) ? 0 : ((px1 - px0) >> res->pdx);
                res->ph = (ry0 == ry1) ? 0 : ((py1 - py0) >> res->pdy);

                if (res->pw * res->ph > maxprec) {
                    maxprec = res->pw * res->ph;
                }
            }
        }

        tccp = &tcp->tccps[0];
        pi[pino].step_p = 1;
        pi[pino].step_c = maxprec * pi[pino].step_p;
        pi[pino].step_r = image->numcomps * pi[pino].step_c;
        pi[pino].step_l = maxres * pi[pino].step_r;

        for (compno = 0; compno < pi->numcomps; compno++) {
            opj_pi_comp_t *comp = &pi->comps[compno];
            for (resno = 0; resno < comp->numresolutions; resno++) {
                int dx, dy;
                opj_pi_resolution_t *res = &comp->resolutions[resno];
                dx = comp->dx * (1 << (res->pdx + comp->numresolutions - 1 - resno));
                dy = comp->dy * (1 << (res->pdy + comp->numresolutions - 1 - resno));
                pi[pino].dx = !pi->dx ? dx : int_min(pi->dx, dx);
                pi[pino].dy = !pi->dy ? dy : int_min(pi->dy, dy);
            }
        }

        if (pino == 0) {
            pi[pino].include = (short int*) opj_calloc(tcp->numlayers * pi[pino].step_l,
                               sizeof(short int));
            if (!pi[pino].include) {
                pi_destroy(pi, cp, tileno);
                return NULL;
            }
        } else {
            pi[pino].include = pi[pino - 1].include;
        }

        /* Generation of boundaries for each prog flag*/
        if (tcp->POC && (cp->cinema || ((!cp->cinema) && (t2_mode == FINAL_PASS)))) {
            tcp->pocs[pino].compS = tcp->pocs[pino].compno0;
            tcp->pocs[pino].compE = tcp->pocs[pino].compno1;
            tcp->pocs[pino].resS = tcp->pocs[pino].resno0;
            tcp->pocs[pino].resE = tcp->pocs[pino].resno1;
            tcp->pocs[pino].layE = tcp->pocs[pino].layno1;
            tcp->pocs[pino].prg  = tcp->pocs[pino].prg1;
            if (pino > 0) {
                tcp->pocs[pino].layS = (tcp->pocs[pino].layE > tcp->pocs[pino - 1].layE) ?
                                       tcp->pocs[pino - 1].layE : 0;
            }
        } else {
            tcp->pocs[pino].compS = 0;
            tcp->pocs[pino].compE = image->numcomps;
            tcp->pocs[pino].resS = 0;
            tcp->pocs[pino].resE = maxres;
            tcp->pocs[pino].layS = 0;
            tcp->pocs[pino].layE = tcp->numlayers;
            tcp->pocs[pino].prg  = tcp->prg;
        }
        tcp->pocs[pino].prcS = 0;
        tcp->pocs[pino].prcE = maxprec;;
        tcp->pocs[pino].txS = pi[pino].tx0;
        tcp->pocs[pino].txE = pi[pino].tx1;
        tcp->pocs[pino].tyS = pi[pino].ty0;
        tcp->pocs[pino].tyE = pi[pino].ty1;
        tcp->pocs[pino].dx = pi[pino].dx;
        tcp->pocs[pino].dy = pi[pino].dy;
    }
    return pi;
}