Commit message: Fixed numerous integer overflow problems in the code for packet iterators
in the JPC decoder.
CVE: CVE-2016-9583
CWE: CWE-125
Vulnerability description: An out-of-bounds heap read vulnerability was found in the jpc_pi_nextpcrl() function of jasper before 2.0.6 when processing crafted input.

Project: jasper
File: jpc_t2cod.c

static int jpc_pi_nextcprl(register jpc_pi_t *pi)
{
	int rlvlno;
	jpc_pirlvl_t *pirlvl;
	jpc_pchg_t *pchg;
	int prchind;
	int prcvind;
	int *prclyrno;
	uint_fast32_t trx0;
	uint_fast32_t try0;
	uint_fast32_t r;
	uint_fast32_t rpx;
	uint_fast32_t rpy;

	pchg = pi->pchg;
	if (!pi->prgvolfirst) {
		goto skip;
	} else {
 		pi->prgvolfirst = 0;
 	}
 
	for (pi->compno = pchg->compnostart, pi->picomp = &pi->picomps[pi->compno];
	  pi->compno < JAS_CAST(int, pchg->compnoend) && pi->compno < pi->numcomps;
	  ++pi->compno, ++pi->picomp) {
 		pirlvl = pi->picomp->pirlvls;
		// Check for the potential for overflow problems.
		if (pirlvl->prcwidthexpn + pi->picomp->numrlvls >
		  JAS_UINTFAST32_NUMBITS - 2 ||
		  pirlvl->prcheightexpn + pi->picomp->numrlvls >
		  JAS_UINTFAST32_NUMBITS - 2) {
			return -1;
		}
 		pi->xstep = pi->picomp->hsamp * (JAS_CAST(uint_fast32_t, 1) <<
 		  (pirlvl->prcwidthexpn + pi->picomp->numrlvls - 1));
 		pi->ystep = pi->picomp->vsamp * (JAS_CAST(uint_fast32_t, 1) <<
		  (pirlvl->prcheightexpn + pi->picomp->numrlvls - 1));
		for (rlvlno = 1, pirlvl = &pi->picomp->pirlvls[1];
		  rlvlno < pi->picomp->numrlvls; ++rlvlno, ++pirlvl) {
			pi->xstep = JAS_MIN(pi->xstep, pi->picomp->hsamp *
			  (JAS_CAST(uint_fast32_t, 1) << (pirlvl->prcwidthexpn +
			  pi->picomp->numrlvls - rlvlno - 1)));
			pi->ystep = JAS_MIN(pi->ystep, pi->picomp->vsamp *
			  (JAS_CAST(uint_fast32_t, 1) << (pirlvl->prcheightexpn +
			  pi->picomp->numrlvls - rlvlno - 1)));
		}
		for (pi->y = pi->ystart; pi->y < pi->yend;
		  pi->y += pi->ystep - (pi->y % pi->ystep)) {
			for (pi->x = pi->xstart; pi->x < pi->xend;
			  pi->x += pi->xstep - (pi->x % pi->xstep)) {
				for (pi->rlvlno = pchg->rlvlnostart,
				  pi->pirlvl = &pi->picomp->pirlvls[pi->rlvlno];
				  pi->rlvlno < pi->picomp->numrlvls && pi->rlvlno <
				  pchg->rlvlnoend; ++pi->rlvlno, ++pi->pirlvl) {
					if (pi->pirlvl->numprcs == 0) {
						continue;
					}
					r = pi->picomp->numrlvls - 1 - pi->rlvlno;
					trx0 = JPC_CEILDIV(pi->xstart, pi->picomp->hsamp << r);
 					try0 = JPC_CEILDIV(pi->ystart, pi->picomp->vsamp << r);
 					rpx = r + pi->pirlvl->prcwidthexpn;
 					rpy = r + pi->pirlvl->prcheightexpn;
					if (((pi->x == pi->xstart &&
					  ((trx0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpx))) ||
 					  !(pi->x % (pi->picomp->hsamp << rpx))) &&
					  ((pi->y == pi->ystart &&
					  ((try0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpy))) ||
 					  !(pi->y % (pi->picomp->vsamp << rpy)))) {
						prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x,
						  pi->picomp->hsamp << r), pi->pirlvl->prcwidthexpn) -
						  JPC_FLOORDIVPOW2(trx0, pi->pirlvl->prcwidthexpn);
						prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y,
						  pi->picomp->vsamp << r), pi->pirlvl->prcheightexpn) -
						  JPC_FLOORDIVPOW2(try0, pi->pirlvl->prcheightexpn);
						pi->prcno = prcvind * pi->pirlvl->numhprcs + prchind;
						assert(pi->prcno < pi->pirlvl->numprcs);
						for (pi->lyrno = 0; pi->lyrno < pi->numlyrs &&
						  pi->lyrno < JAS_CAST(int, pchg->lyrnoend);
						  ++pi->lyrno) {
 							prclyrno = &pi->pirlvl->prclyrnos[pi->prcno];
 							if (pi->lyrno >= *prclyrno) {
 								++(*prclyrno);
								return 0;
							}
skip:
							;
						}
					}
				}
			}
		}
	}
	return 1;
}

Project: jasper
File: jpc_t2cod.c

static int jpc_pi_nextrpcl(register jpc_pi_t *pi)
{
	int rlvlno;
	jpc_pirlvl_t *pirlvl;
	jpc_pchg_t *pchg;
	int prchind;
	int prcvind;
	int *prclyrno;
	int compno;
	jpc_picomp_t *picomp;
	int xstep;
	int ystep;
	uint_fast32_t r;
	uint_fast32_t rpx;
	uint_fast32_t rpy;
	uint_fast32_t trx0;
	uint_fast32_t try0;

	pchg = pi->pchg;
	if (!pi->prgvolfirst) {
		goto skip;
	} else {
		pi->xstep = 0;
		pi->ystep = 0;
		for (compno = 0, picomp = pi->picomps; compno < pi->numcomps;
 		  ++compno, ++picomp) {
 			for (rlvlno = 0, pirlvl = picomp->pirlvls; rlvlno <
 			  picomp->numrlvls; ++rlvlno, ++pirlvl) {
				// Check for the potential for overflow problems.
				if (pirlvl->prcwidthexpn + pi->picomp->numrlvls >
				  JAS_UINTFAST32_NUMBITS - 2 ||
				  pirlvl->prcheightexpn + pi->picomp->numrlvls >
				  JAS_UINTFAST32_NUMBITS - 2) {
					return -1;
				}
				xstep = picomp->hsamp * (JAS_CAST(uint_fast32_t, 1) <<
				  (pirlvl->prcwidthexpn + picomp->numrlvls - rlvlno - 1));
				ystep = picomp->vsamp * (JAS_CAST(uint_fast32_t, 1) <<
				  (pirlvl->prcheightexpn + picomp->numrlvls - rlvlno - 1));
 				pi->xstep = (!pi->xstep) ? xstep : JAS_MIN(pi->xstep, xstep);
 				pi->ystep = (!pi->ystep) ? ystep : JAS_MIN(pi->ystep, ystep);
 			}
		}
		pi->prgvolfirst = 0;
	}

	for (pi->rlvlno = pchg->rlvlnostart; pi->rlvlno < pchg->rlvlnoend &&
	  pi->rlvlno < pi->maxrlvls; ++pi->rlvlno) {
		for (pi->y = pi->ystart; pi->y < pi->yend; pi->y +=
		  pi->ystep - (pi->y % pi->ystep)) {
			for (pi->x = pi->xstart; pi->x < pi->xend; pi->x +=
			  pi->xstep - (pi->x % pi->xstep)) {
				for (pi->compno = pchg->compnostart,
				  pi->picomp = &pi->picomps[pi->compno];
				  pi->compno < JAS_CAST(int, pchg->compnoend) && pi->compno <
				  pi->numcomps; ++pi->compno, ++pi->picomp) {
					if (pi->rlvlno >= pi->picomp->numrlvls) {
						continue;
					}
					pi->pirlvl = &pi->picomp->pirlvls[pi->rlvlno];
					if (pi->pirlvl->numprcs == 0) {
						continue;
					}
					r = pi->picomp->numrlvls - 1 - pi->rlvlno;
					rpx = r + pi->pirlvl->prcwidthexpn;
 					rpy = r + pi->pirlvl->prcheightexpn;
 					trx0 = JPC_CEILDIV(pi->xstart, pi->picomp->hsamp << r);
 					try0 = JPC_CEILDIV(pi->ystart, pi->picomp->vsamp << r);
					if (((pi->x == pi->xstart &&
					  ((trx0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpx)))
					  || !(pi->x % (JAS_CAST(uint_fast32_t, 1) << rpx))) &&
					  ((pi->y == pi->ystart &&
					  ((try0 << r) % (JAS_CAST(uint_fast32_t, 1) << rpy)))
					  || !(pi->y % (JAS_CAST(uint_fast32_t, 1) << rpy)))) {
						prchind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->x,
						  pi->picomp->hsamp << r), pi->pirlvl->prcwidthexpn) -
						  JPC_FLOORDIVPOW2(trx0, pi->pirlvl->prcwidthexpn);
						prcvind = JPC_FLOORDIVPOW2(JPC_CEILDIV(pi->y,
						  pi->picomp->vsamp << r), pi->pirlvl->prcheightexpn) -
						  JPC_FLOORDIVPOW2(try0, pi->pirlvl->prcheightexpn);
 						pi->prcno = prcvind * pi->pirlvl->numhprcs + prchind;
 
 						assert(pi->prcno < pi->pirlvl->numprcs);
 						for (pi->lyrno = 0; pi->lyrno <
						  pi->numlyrs && pi->lyrno < JAS_CAST(int,
						  pchg->lyrnoend); ++pi->lyrno) {
 							prclyrno = &pi->pirlvl->prclyrnos[pi->prcno];
 							if (pi->lyrno >= *prclyrno) {
 								++(*prclyrno);
								return 0;
							}
skip:
							;
						}
					}
				}
			}
		}
	}
	return 1;
}