Commit message: Fixed numerous integer overflow problems in the code for packet iterators
in the JPC decoder.
CVE: CVE-2016-9583
CWE: CWE-125
Vulnerability description: An out-of-bounds heap read vulnerability was found in the jpc_pi_nextpcrl() function of jasper before 2.0.6 when processing crafted input.

Project: jasper
File: jpc_t2cod.c

static int jpc_pi_nextpcrl(register jpc_pi_t *pi)
{
	int rlvlno;
	jpc_pirlvl_t *pirlvl;
	jpc_pchg_t *pchg;
	int prchind;
	int prcvind;
	int *prclyrno;
	int compno;
	jpc_picomp_t *picomp;
	int xstep;
	int ystep;
	uint_fast32_t trx0;
	uint_fast32_t try0;
	uint_fast32_t r;
	uint_fast32_t rpx;
	uint_fast32_t rpy;

	pchg = pi->pchg;
	if (!pi->prgvolfirst) {
		goto skip;
	} else {
		pi->xstep = 0;
		pi->ystep = 0;
		for (compno = 0, picomp = pi->picomps; compno < pi->numcomps;
 		  ++compno, ++picomp) {
 			for (rlvlno = 0, pirlvl = picomp->pirlvls; rlvlno <
 			  picomp->numrlvls; ++rlvlno, ++pirlvl) {
				// Check for the potential for overflow problems.
				if (pirlvl->prcwidthexpn + pi->picomp->numrlvls >
				  JAS_UINTFAST32_NUMBITS - 2 ||
				  pirlvl->prcheightexpn + pi->picomp->numrlvls >
				  JAS_UINTFAST32_NUMBITS - 2) {
					return -1;
				}
				xstep = picomp->hsamp * (JAS_CAST(uint_fast32_t, 1) <<
				  (pirlvl->prcwidthexpn + picomp->numrlvls - rlvlno - 1));
				ystep = picomp->vsamp * (JAS_CAST(uint_fast32_t, 1) <<
				  (pirlvl->prcheightexpn + picomp->numrlvls - rlvlno - 1));
				pi->xstep = (!pi->xstep) ? xstep : JAS_MIN(pi->xstep, xstep);
				pi->ystep = (!pi->ystep) ? ystep : JAS_MIN(pi->ystep, ystep);
 			}
 		}
 		pi->prgvolfirst = 0;
	}

	for (pi->y = pi->ystart; pi->y < pi->yend; pi->y += pi->ystep -
	  (pi->y % pi->ystep)) {
		for (pi->x = pi->xstart; pi->x < pi->xend; pi->x += pi->xstep -
		  (pi->x % pi->xstep)) {
			for (pi->compno = pchg->compnostart, pi->picomp =
			  &pi->picomps[pi->compno]; pi->compno < pi->numcomps
			  && pi->compno < JAS_CAST(int, pchg->compnoend); ++pi->compno,
			  ++pi->picomp) {
				for (pi->rlvlno = pchg->rlvlnostart,
				  pi->pirlvl = &pi->picomp->pirlvls[pi->rlvlno];
				  pi->rlvlno < pi->picomp->numrlvls &&
				  pi->rlvlno < pchg->rlvlnoend; ++pi->rlvlno,
				  ++pi->pirlvl) {
					if (pi->pirlvl->numprcs == 0) {
						continue;
					}
					r = pi->picomp->numrlvls - 1 - pi->rlvlno;
					trx0 = JPC_CEILDIV(pi->xstart, pi->picomp->hsamp << r);
 					try0 = JPC_CEILDIV(pi->ystart, pi->picomp->vsamp << r);
 					rpx = r + pi->pirlvl->prcwidthexpn;
 					rpy = r + pi->pirlvl->prcheightexpn;
					if (((pi->x == pi->xstart &&
					  ((trx0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpx))) ||
 					  !(pi->x % (pi->picomp->hsamp << rpx))) &&
					  ((pi->y == pi->ystart &&
					  ((try0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpy))) ||
 					  !(pi->y % (pi->picomp->vsamp << rpy)))) {
						prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x,
						  pi->picomp->hsamp << r), pi->pirlvl->prcwidthexpn) -
						  JPC_FLOORDIVPOW2(trx0, pi->pirlvl->prcwidthexpn);
						prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y,
						  pi->picomp->vsamp << r), pi->pirlvl->prcheightexpn) -
						  JPC_FLOORDIVPOW2(try0, pi->pirlvl->prcheightexpn);
 						pi->prcno = prcvind * pi->pirlvl->numhprcs + prchind;
 						assert(pi->prcno < pi->pirlvl->numprcs);
 						for (pi->lyrno = 0; pi->lyrno < pi->numlyrs &&
						  pi->lyrno < JAS_CAST(int, pchg->lyrnoend);
						  ++pi->lyrno) {
 							prclyrno = &pi->pirlvl->prclyrnos[pi->prcno];
 							if (pi->lyrno >= *prclyrno) {
 								++(*prclyrno);
								return 0;
							}
skip:
							;
						}
					}
				}
			}
		}
	}
	return 1;
}