Commit message: jpc_dec: fix another integer overflow in SIZ

Validate the width and height values first, before doing anything
else.  This prevents integer overflows in the `numhtiles` /
`numvtiles` calculation below, triggering assertion failures.

Fixes CVE-2017-13750

Closes https://github.com/mdadams/jasper/issues/165
Closes https://github.com/mdadams/jasper/issues/174
CVE: CVE-2017-13746
CWE: CWE-617
Vulnerability description: There is a reachable assertion abort in the function jpc_dec_process_siz() in jpc/jpc_dec.c:1297 in JasPer 2.0.12 that will lead to a remote denial of service attack.

Project: jasper
File: jpc_dec.c

static int jpc_dec_tilefini(jpc_dec_t *dec, jpc_dec_tile_t *tile)
{
	jpc_dec_tcomp_t *tcomp;
	int compno;
	int bandno;
	unsigned rlvlno;
	jpc_dec_band_t *band;
	jpc_dec_rlvl_t *rlvl;
	int prcno;
	jpc_dec_prc_t *prc;
	jpc_dec_seg_t *seg;
	jpc_dec_cblk_t *cblk;
	int cblkno;

	if (tile->tcomps) {

		for (compno = 0, tcomp = tile->tcomps; compno < dec->numcomps;
		  ++compno, ++tcomp) {
			for (rlvlno = 0, rlvl = tcomp->rlvls; rlvlno < tcomp->numrlvls;
			  ++rlvlno, ++rlvl) {
				if (!rlvl->bands) {
					continue;
				}
				for (bandno = 0, band = rlvl->bands; bandno < rlvl->numbands;
				  ++bandno, ++band) {
					if (band->prcs) {
						for (prcno = 0, prc = band->prcs; prcno <
						  rlvl->numprcs; ++prcno, ++prc) {
							if (!prc->cblks) {
								continue;
							}
							for (cblkno = 0, cblk = prc->cblks; cblkno <
							  prc->numcblks; ++cblkno, ++cblk) {

								while (cblk->segs.head) {
									seg = cblk->segs.head;
									jpc_seglist_remove(&cblk->segs, seg);
									jpc_seg_destroy(seg);
								}
								jas_matrix_destroy(cblk->data);
								if (cblk->mqdec) {
									jpc_mqdec_destroy(cblk->mqdec);
								}
								if (cblk->nulldec) {
									jpc_bitstream_close(cblk->nulldec);
								}
								if (cblk->flags) {
									jas_matrix_destroy(cblk->flags);
								}
							}
							if (prc->incltagtree) {
								jpc_tagtree_destroy(prc->incltagtree);
							}
							if (prc->numimsbstagtree) {
								jpc_tagtree_destroy(prc->numimsbstagtree);
							}
							if (prc->cblks) {
								jas_free(prc->cblks);
							}
						}
					}
					if (band->data) {
						jas_matrix_destroy(band->data);
					}
					if (band->prcs) {
						jas_free(band->prcs);
					}
				}
				if (rlvl->bands) {
					jas_free(rlvl->bands);
				}
			}
			if (tcomp->rlvls) {
				jas_free(tcomp->rlvls);
			}
			if (tcomp->data) {
				jas_matrix_destroy(tcomp->data);
			}
			if (tcomp->tsfb) {
				jpc_tsfb_destroy(tcomp->tsfb);
			}
		}
	}

	if (tile->cp) {
		jpc_dec_cp_destroy(tile->cp);
		//tile->cp = 0;
	}
	if (tile->tcomps) {
		jas_free(tile->tcomps);
		//tile->tcomps = 0;
	}
	if (tile->pi) {
		jpc_pi_destroy(tile->pi);
		//tile->pi = 0;
	}
	if (tile->pkthdrstream) {
		jas_stream_close(tile->pkthdrstream);
		//tile->pkthdrstream = 0;
	}
	if (tile->pptstab) {
		jpc_ppxstab_destroy(tile->pptstab);
		//tile->pptstab = 0;
	}

	tile->state = JPC_TILE_DONE;

	return 0;
}

Project: jasper
File: jpc_dec.c

static int jpc_dec_tiledecode(jpc_dec_t *dec, jpc_dec_tile_t *tile)
{
	int i;
	int j;
	jpc_dec_tcomp_t *tcomp;
	jpc_dec_rlvl_t *rlvl;
	jpc_dec_band_t *band;
	int compno;
	unsigned rlvlno;
	int bandno;
	int adjust;
	int v;
	jpc_dec_ccp_t *ccp;
	jpc_dec_cmpt_t *cmpt;

	if (jpc_dec_decodecblks(dec, tile)) {
		jas_eprintf("jpc_dec_decodecblks failed\n");
		return -1;
	}

	/* Perform dequantization. */
	for (compno = 0, tcomp = tile->tcomps; compno < dec->numcomps;
	  ++compno, ++tcomp) {
		ccp = &tile->cp->ccps[compno];
		for (rlvlno = 0, rlvl = tcomp->rlvls; rlvlno < tcomp->numrlvls;
		  ++rlvlno, ++rlvl) {
			if (!rlvl->bands) {
				continue;
			}
			for (bandno = 0, band = rlvl->bands;
			  bandno < rlvl->numbands; ++bandno, ++band) {
				if (!band->data) {
					continue;
				}
				jpc_undo_roi(band->data, band->roishift, ccp->roishift -
				  band->roishift, band->numbps);
				if (tile->realmode) {
					jas_matrix_asl(band->data, JPC_FIX_FRACBITS);
					jpc_dequantize(band->data, band->absstepsize);
				}

			}
		}
	}

	/* Apply an inverse wavelet transform if necessary. */
	for (compno = 0, tcomp = tile->tcomps; compno < dec->numcomps;
	  ++compno, ++tcomp) {
		ccp = &tile->cp->ccps[compno];
		jpc_tsfb_synthesize(tcomp->tsfb, tcomp->data);
	}


	/* Apply an inverse intercomponent transform if necessary. */
	switch (tile->cp->mctid) {
	case JPC_MCT_RCT:
		if (dec->numcomps < 3) {
			jas_eprintf("RCT requires at least three components\n");
			return -1;
		}
		if (!jas_image_cmpt_domains_same(dec->image)) {
			jas_eprintf("RCT requires all components have the same domain\n");
			return -1;
		}
		jpc_irct(tile->tcomps[0].data, tile->tcomps[1].data,
		  tile->tcomps[2].data);
		break;
	case JPC_MCT_ICT:
		if (dec->numcomps < 3) {
			jas_eprintf("ICT requires at least three components\n");
			return -1;
		}
		if (!jas_image_cmpt_domains_same(dec->image)) {
			jas_eprintf("RCT requires all components have the same domain\n");
			return -1;
		}
		jpc_iict(tile->tcomps[0].data, tile->tcomps[1].data,
		  tile->tcomps[2].data);
		break;
	}

	/* Perform rounding and convert to integer values. */
	if (tile->realmode) {
		for (compno = 0, tcomp = tile->tcomps; compno < dec->numcomps;
		  ++compno, ++tcomp) {
			for (i = 0; i < jas_matrix_numrows(tcomp->data); ++i) {
				for (j = 0; j < jas_matrix_numcols(tcomp->data); ++j) {
					v = jas_matrix_get(tcomp->data, i, j);
					v = jpc_fix_round(v);
					jas_matrix_set(tcomp->data, i, j, jpc_fixtoint(v));
				}
			}
		}
	}

	/* Perform level shift. */
	for (compno = 0, tcomp = tile->tcomps, cmpt = dec->cmpts; compno <
	  dec->numcomps; ++compno, ++tcomp, ++cmpt) {
		adjust = cmpt->sgnd ? 0 : (1 << (cmpt->prec - 1));
		for (i = 0; i < jas_matrix_numrows(tcomp->data); ++i) {
			for (j = 0; j < jas_matrix_numcols(tcomp->data); ++j) {
				*jas_matrix_getref(tcomp->data, i, j) += adjust;
			}
		}
	}

	/* Perform clipping. */
	for (compno = 0, tcomp = tile->tcomps, cmpt = dec->cmpts; compno <
	  dec->numcomps; ++compno, ++tcomp, ++cmpt) {
		jpc_fix_t mn;
		jpc_fix_t mx;
		mn = cmpt->sgnd ? (-(1 << (cmpt->prec - 1))) : (0);
		mx = cmpt->sgnd ? ((1 << (cmpt->prec - 1)) - 1) : ((1 <<
		  cmpt->prec) - 1);
		jas_matrix_clip(tcomp->data, mn, mx);
	}

	/* XXX need to free tsfb struct */

	/* Write the data for each component of the image. */
	for (compno = 0, tcomp = tile->tcomps, cmpt = dec->cmpts; compno <
	  dec->numcomps; ++compno, ++tcomp, ++cmpt) {
		if (jas_image_writecmpt(dec->image, compno, tcomp->xstart -
		  JPC_CEILDIV(dec->xstart, cmpt->hstep), tcomp->ystart -
		  JPC_CEILDIV(dec->ystart, cmpt->vstep), jas_matrix_numcols(
		  tcomp->data), jas_matrix_numrows(tcomp->data), tcomp->data)) {
			jas_eprintf("write component failed\n");
			return -1;
		}
	}

	return 0;
}

Project: jasper
File: jpc_dec.c

static void jpc_dec_destroy(jpc_dec_t *dec)
{
	if (dec->cstate) {
		jpc_cstate_destroy(dec->cstate);
	}
	if (dec->pkthdrstreams) {
		jpc_streamlist_destroy(dec->pkthdrstreams);
	}
	if (dec->ppmstab) {
		jpc_ppxstab_destroy(dec->ppmstab);
	}
	if (dec->image) {
		jas_image_destroy(dec->image);
	}

	if (dec->cp) {
		jpc_dec_cp_destroy(dec->cp);
	}

	if (dec->cmpts) {
		jas_free(dec->cmpts);
	}

	if (dec->tiles) {
		int tileno;
		jpc_dec_tile_t *tile;

		for (tileno = 0, tile = dec->tiles; tileno < dec->numtiles; ++tileno,
		  ++tile) {
			if (tile->state != JPC_TILE_DONE) {
				jpc_dec_tilefini(dec, tile);
			}
		}

		jas_free(dec->tiles);
	}

	jas_free(dec);
}

Project: jasper
File: jpc_dec.c

static int jpc_dec_tileinit(jpc_dec_t *dec, jpc_dec_tile_t *tile)
{
	jpc_dec_tcomp_t *tcomp;
	int compno;
	unsigned rlvlno;
	jpc_dec_rlvl_t *rlvl;
	jpc_dec_band_t *band;
	jpc_dec_prc_t *prc;
	int bndno;
	jpc_tsfb_band_t *bnd;
	int bandno;
	jpc_dec_ccp_t *ccp;
	int prccnt;
	jpc_dec_cblk_t *cblk;
	int cblkcnt;
	uint_fast32_t tlprcxstart;
	uint_fast32_t tlprcystart;
	uint_fast32_t brprcxend;
	uint_fast32_t brprcyend;
	uint_fast32_t tlcbgxstart;
	uint_fast32_t tlcbgystart;
	uint_fast32_t brcbgxend;
	uint_fast32_t brcbgyend;
	uint_fast32_t cbgxstart;
	uint_fast32_t cbgystart;
	uint_fast32_t cbgxend;
	uint_fast32_t cbgyend;
	uint_fast32_t tlcblkxstart;
	uint_fast32_t tlcblkystart;
	uint_fast32_t brcblkxend;
	uint_fast32_t brcblkyend;
	uint_fast32_t cblkxstart;
	uint_fast32_t cblkystart;
	uint_fast32_t cblkxend;
	uint_fast32_t cblkyend;
	uint_fast32_t tmpxstart;
	uint_fast32_t tmpystart;
	uint_fast32_t tmpxend;
	uint_fast32_t tmpyend;
	jpc_dec_cp_t *cp;
	jpc_tsfb_band_t bnds[JPC_MAXBANDS];
	jpc_pchg_t *pchg;
	int pchgno;
	jpc_dec_cmpt_t *cmpt;

	cp = tile->cp;
	tile->realmode = 0;
	if (cp->mctid == JPC_MCT_ICT) {
		tile->realmode = 1;
	}

	for (compno = 0, tcomp = tile->tcomps, cmpt = dec->cmpts; compno <
	  dec->numcomps; ++compno, ++tcomp, ++cmpt) {
		ccp = &tile->cp->ccps[compno];
		if (ccp->qmfbid == JPC_COX_INS) {
			tile->realmode = 1;
		}
		tcomp->numrlvls = ccp->numrlvls;
		if (!(tcomp->rlvls = jas_alloc2(tcomp->numrlvls,
		  sizeof(jpc_dec_rlvl_t)))) {
			return -1;
		}
		for (rlvlno = 0, rlvl = tcomp->rlvls; rlvlno < tcomp->numrlvls;
		  ++rlvlno, ++rlvl) {
			rlvl->bands = NULL;
		}
		if (!(tcomp->data = jas_seq2d_create(JPC_CEILDIV(tile->xstart,
		  cmpt->hstep), JPC_CEILDIV(tile->ystart, cmpt->vstep),
		  JPC_CEILDIV(tile->xend, cmpt->hstep), JPC_CEILDIV(tile->yend,
		  cmpt->vstep)))) {
			return -1;
		}
		if (!(tcomp->tsfb = jpc_cod_gettsfb(ccp->qmfbid,
		  tcomp->numrlvls - 1))) {
			return -1;
		}
		{
			jpc_tsfb_getbands(tcomp->tsfb, jas_seq2d_xstart(tcomp->data),
			  jas_seq2d_ystart(tcomp->data), jas_seq2d_xend(tcomp->data),
			  jas_seq2d_yend(tcomp->data), bnds);
		}
		for (rlvlno = 0, rlvl = tcomp->rlvls; rlvlno < tcomp->numrlvls;
		  ++rlvlno, ++rlvl) {
			rlvl->bands = 0;
			rlvl->xstart = JPC_CEILDIVPOW2(tcomp->xstart,
			  tcomp->numrlvls - 1 - rlvlno);
			rlvl->ystart = JPC_CEILDIVPOW2(tcomp->ystart,
			  tcomp->numrlvls - 1 - rlvlno);
			rlvl->xend = JPC_CEILDIVPOW2(tcomp->xend,
			  tcomp->numrlvls - 1 - rlvlno);
			rlvl->yend = JPC_CEILDIVPOW2(tcomp->yend,
			  tcomp->numrlvls - 1 - rlvlno);
			rlvl->prcwidthexpn = ccp->prcwidthexpns[rlvlno];
			rlvl->prcheightexpn = ccp->prcheightexpns[rlvlno];
			tlprcxstart = JPC_FLOORDIVPOW2(rlvl->xstart,
			  rlvl->prcwidthexpn) << rlvl->prcwidthexpn;
			tlprcystart = JPC_FLOORDIVPOW2(rlvl->ystart,
			  rlvl->prcheightexpn) << rlvl->prcheightexpn;
			brprcxend = JPC_CEILDIVPOW2(rlvl->xend,
			  rlvl->prcwidthexpn) << rlvl->prcwidthexpn;
			brprcyend = JPC_CEILDIVPOW2(rlvl->yend,
			  rlvl->prcheightexpn) << rlvl->prcheightexpn;
			rlvl->numhprcs = (brprcxend - tlprcxstart) >>
			  rlvl->prcwidthexpn;
			rlvl->numvprcs = (brprcyend - tlprcystart) >>
			  rlvl->prcheightexpn;
			rlvl->numprcs = rlvl->numhprcs * rlvl->numvprcs;

			if (rlvl->numprcs >= 64 * 1024) {
				/* avoid out-of-memory due to
				   malicious file; this limit is
				   rather arbitrary; "good" files I
				   have seen have values 1..12 */
				return -1;
			}

			if (rlvl->xstart >= rlvl->xend || rlvl->ystart >= rlvl->yend) {
				rlvl->bands = 0;
				rlvl->numprcs = 0;
				rlvl->numhprcs = 0;
				rlvl->numvprcs = 0;
				continue;
			}	
			if (!rlvlno) {
				tlcbgxstart = tlprcxstart;
				tlcbgystart = tlprcystart;
				brcbgxend = brprcxend;
				brcbgyend = brprcyend;
				rlvl->cbgwidthexpn = rlvl->prcwidthexpn;
				rlvl->cbgheightexpn = rlvl->prcheightexpn;
			} else {
				tlcbgxstart = JPC_CEILDIVPOW2(tlprcxstart, 1);
				tlcbgystart = JPC_CEILDIVPOW2(tlprcystart, 1);
				brcbgxend = JPC_CEILDIVPOW2(brprcxend, 1);
				brcbgyend = JPC_CEILDIVPOW2(brprcyend, 1);
				rlvl->cbgwidthexpn = rlvl->prcwidthexpn - 1;
				rlvl->cbgheightexpn = rlvl->prcheightexpn - 1;
				if (rlvl->cbgwidthexpn < 0 || rlvl->cbgheightexpn < 0) {
					/* the control block width/height offset
					   exponent must not be negative */
					return -1;
				}
			}
			rlvl->cblkwidthexpn = JAS_MIN(ccp->cblkwidthexpn,
			  rlvl->cbgwidthexpn);
			rlvl->cblkheightexpn = JAS_MIN(ccp->cblkheightexpn,
			  rlvl->cbgheightexpn);

			rlvl->numbands = (!rlvlno) ? 1 : 3;
			if (!(rlvl->bands = jas_alloc2(rlvl->numbands,
			  sizeof(jpc_dec_band_t)))) {
				return -1;
			}
			for (bandno = 0, band = rlvl->bands;
			  bandno < rlvl->numbands; ++bandno, ++band) {
				bndno = (!rlvlno) ? 0 : (3 * (rlvlno - 1) +
				  bandno + 1);
				bnd = &bnds[bndno];

				band->orient = bnd->orient;
				band->stepsize = ccp->stepsizes[bndno];
				band->analgain = JPC_NOMINALGAIN(ccp->qmfbid,
				  tcomp->numrlvls - 1, rlvlno, band->orient);
				band->absstepsize = jpc_calcabsstepsize(band->stepsize,
				  cmpt->prec + band->analgain);
				band->numbps = ccp->numguardbits +
				  JPC_QCX_GETEXPN(band->stepsize) - 1;
				band->roishift = (ccp->roishift + band->numbps >= JPC_PREC) ?
				  (JPC_PREC - 1 - band->numbps) : ccp->roishift;
				band->data = 0;
				band->prcs = 0;
				if (bnd->xstart == bnd->xend || bnd->ystart == bnd->yend) {
					continue;
				}
				if (!(band->data = jas_seq2d_create(0, 0, 0, 0))) {
					return -1;
				}
				if (jas_seq2d_bindsub(band->data, tcomp->data, bnd->locxstart,
						      bnd->locystart, bnd->locxend, bnd->locyend)) {
					return -1;
				}
				jas_seq2d_setshift(band->data, bnd->xstart, bnd->ystart);

				assert(rlvl->numprcs);

				if (!(band->prcs = jas_alloc2(rlvl->numprcs,
				  sizeof(jpc_dec_prc_t)))) {
					return -1;
				}

/************************************************/
				cbgxstart = tlcbgxstart;
				cbgystart = tlcbgystart;
				for (prccnt = rlvl->numprcs, prc = band->prcs;
				  prccnt > 0; --prccnt, ++prc) {
					cbgxend = cbgxstart + (1 << rlvl->cbgwidthexpn);
					cbgyend = cbgystart + (1 << rlvl->cbgheightexpn);
					prc->xstart = JAS_MAX(cbgxstart, JAS_CAST(uint_fast32_t,
					  jas_seq2d_xstart(band->data)));
					prc->ystart = JAS_MAX(cbgystart, JAS_CAST(uint_fast32_t,
					  jas_seq2d_ystart(band->data)));
					prc->xend = JAS_MIN(cbgxend, JAS_CAST(uint_fast32_t,
					  jas_seq2d_xend(band->data)));
					prc->yend = JAS_MIN(cbgyend, JAS_CAST(uint_fast32_t,
					  jas_seq2d_yend(band->data)));
					if (prc->xend > prc->xstart && prc->yend > prc->ystart) {
						tlcblkxstart = JPC_FLOORDIVPOW2(prc->xstart,
						  rlvl->cblkwidthexpn) << rlvl->cblkwidthexpn;
						tlcblkystart = JPC_FLOORDIVPOW2(prc->ystart,
						  rlvl->cblkheightexpn) << rlvl->cblkheightexpn;
						brcblkxend = JPC_CEILDIVPOW2(prc->xend,
						  rlvl->cblkwidthexpn) << rlvl->cblkwidthexpn;
						brcblkyend = JPC_CEILDIVPOW2(prc->yend,
						  rlvl->cblkheightexpn) << rlvl->cblkheightexpn;
						prc->numhcblks = (brcblkxend - tlcblkxstart) >>
						  rlvl->cblkwidthexpn;
						prc->numvcblks = (brcblkyend - tlcblkystart) >>
						  rlvl->cblkheightexpn;
						prc->numcblks = prc->numhcblks * prc->numvcblks;
						assert(prc->numcblks > 0);

						if (!(prc->incltagtree = jpc_tagtree_create(
						  prc->numhcblks, prc->numvcblks))) {
							return -1;
						}
						if (!(prc->numimsbstagtree = jpc_tagtree_create(
						  prc->numhcblks, prc->numvcblks))) {
							return -1;
						}
						if (!(prc->cblks = jas_alloc2(prc->numcblks,
						  sizeof(jpc_dec_cblk_t)))) {
							return -1;
						}

						cblkxstart = cbgxstart;
						cblkystart = cbgystart;
						for (cblkcnt = prc->numcblks, cblk = prc->cblks;
						  cblkcnt > 0;) {
							cblkxend = cblkxstart + (1 << rlvl->cblkwidthexpn);
							cblkyend = cblkystart + (1 << rlvl->cblkheightexpn);
							tmpxstart = JAS_MAX(cblkxstart, prc->xstart);
							tmpystart = JAS_MAX(cblkystart, prc->ystart);
							tmpxend = JAS_MIN(cblkxend, prc->xend);
							tmpyend = JAS_MIN(cblkyend, prc->yend);
							if (tmpxend > tmpxstart && tmpyend > tmpystart) {
								cblk->firstpassno = -1;
								cblk->mqdec = 0;
								cblk->nulldec = 0;
								cblk->flags = 0;
								cblk->numpasses = 0;
								cblk->segs.head = 0;
								cblk->segs.tail = 0;
								cblk->curseg = 0;
								cblk->numimsbs = 0;
								cblk->numlenbits = 3;
								cblk->flags = 0;
								if (!(cblk->data = jas_seq2d_create(0, 0, 0,
								  0))) {
									return -1;
								}
								if (jas_seq2d_bindsub(cblk->data, band->data,
										      tmpxstart, tmpystart, tmpxend, tmpyend)) {
									return -1;
								}
								++cblk;
								--cblkcnt;
							}
							cblkxstart += 1 << rlvl->cblkwidthexpn;
							if (cblkxstart >= cbgxend) {
								cblkxstart = cbgxstart;
								cblkystart += 1 << rlvl->cblkheightexpn;
							}
						}

					} else {
						prc->cblks = 0;
						prc->incltagtree = 0;
						prc->numimsbstagtree = 0;
					}
					cbgxstart += 1 << rlvl->cbgwidthexpn;
					if (cbgxstart >= brcbgxend) {
						cbgxstart = tlcbgxstart;
						cbgystart += 1 << rlvl->cbgheightexpn;
					}

				}
/********************************************/
			}
		}
	}

	if (!(tile->pi = jpc_dec_pi_create(dec, tile))) {
		return -1;
	}

	for (pchgno = 0; pchgno < jpc_pchglist_numpchgs(tile->cp->pchglist);
	  ++pchgno) {
		pchg = jpc_pchg_copy(jpc_pchglist_get(tile->cp->pchglist, pchgno));
		assert(pchg);
		jpc_pi_addpchg(tile->pi, pchg);
	}
	jpc_pi_init(tile->pi);

	return 0;
}

Project: jasper
File: jpc_dec.c

static int jpc_dec_process_siz(jpc_dec_t *dec, jpc_ms_t *ms)
{
	jpc_siz_t *siz = &ms->parms.siz;
	int compno;
	int tileno;
	jpc_dec_tile_t *tile;
	jpc_dec_tcomp_t *tcomp;
	int htileno;
	int vtileno;
	jpc_dec_cmpt_t *cmpt;
	size_t size;
	size_t num_samples;
	size_t num_samples_delta;

	size_t total_samples;
	if (!jas_safe_size_mul(siz->width, siz->height, &total_samples) ||
	    (dec->max_samples > 0 && total_samples > dec->max_samples)) {
		jas_eprintf("image too large\n");
		return -1;
	}

	size_t tile_samples;
	if (!jas_safe_size_mul(siz->tilewidth, siz->tileheight, &tile_samples) ||
	    (dec->max_samples > 0 && tile_samples > dec->max_samples)) {
		jas_eprintf("tile too large\n");
		return -1;
	}

	dec->xstart = siz->xoff;
	dec->ystart = siz->yoff;
	dec->xend = siz->width;
	dec->yend = siz->height;
	dec->tilewidth = siz->tilewidth;
	dec->tileheight = siz->tileheight;
	dec->tilexoff = siz->tilexoff;
	dec->tileyoff = siz->tileyoff;
	dec->numcomps = siz->numcomps;

	if (!(dec->cp = jpc_dec_cp_create(dec->numcomps))) {
		return -1;
	}

	if (!(dec->cmpts = jas_alloc2(dec->numcomps, sizeof(jpc_dec_cmpt_t)))) {
		return -1;
	}

	num_samples = 0;
	for (compno = 0, cmpt = dec->cmpts; compno < dec->numcomps; ++compno,
	  ++cmpt) {
		cmpt->prec = siz->comps[compno].prec;
		cmpt->sgnd = siz->comps[compno].sgnd;
		cmpt->hstep = siz->comps[compno].hsamp;
		cmpt->vstep = siz->comps[compno].vsamp;
		cmpt->width = JPC_CEILDIV(dec->xend, cmpt->hstep) -
		  JPC_CEILDIV(dec->xstart, cmpt->hstep);
		cmpt->height = JPC_CEILDIV(dec->yend, cmpt->vstep) -
		  JPC_CEILDIV(dec->ystart, cmpt->vstep);
		cmpt->hsubstep = 0;
		cmpt->vsubstep = 0;

		if (!jas_safe_size_mul(cmpt->width, cmpt->height, &num_samples_delta)) {
			jas_eprintf("image too large\n");
			return -1;
		}
		if (!jas_safe_size_add(num_samples, num_samples_delta, &num_samples)) {
			jas_eprintf("image too large\n");
			return -1;
		}
	}

	if (dec->max_samples > 0 && num_samples > dec->max_samples) {
		jas_eprintf("maximum number of samples exceeded (%zu > %zu)\n",
		  num_samples, dec->max_samples);
		return -1;
	}

	dec->image = 0;

	dec->numhtiles = JPC_CEILDIV(dec->xend - dec->tilexoff, dec->tilewidth);
	dec->numvtiles = JPC_CEILDIV(dec->yend - dec->tileyoff, dec->tileheight);
	assert(dec->numhtiles >= 0);
	assert(dec->numvtiles >= 0);
	if (!jas_safe_size_mul(dec->numhtiles, dec->numvtiles, &size) ||
	  size > INT_MAX) {
		return -1;
	}
	if (dec->max_samples > 0 && size > dec->max_samples / 16 / 16) {
		/* avoid Denial of Service by a malicious input file
		   with millions of tiny tiles; if max_samples is
		   configured, then assume the tiles are at least
		   16x16, and don't allow more than this number of
		   tiles */
		return -1;
	}
	if (dec->max_samples > 0 && size > dec->max_samples / dec->numcomps / 16) {
		/* another DoS check: since each tile allocates an
		   array of components, this check attempts to catch
		   excessive tile*component numbers */
		return -1;
	}
	dec->numtiles = size;
	JAS_DBGLOG(10, ("numtiles = %d; numhtiles = %d; numvtiles = %d;\n",
	  dec->numtiles, dec->numhtiles, dec->numvtiles));
	if (!(dec->tiles = jas_alloc2(dec->numtiles, sizeof(jpc_dec_tile_t)))) {
		return -1;
	}

	for (tileno = 0, tile = dec->tiles; tileno < dec->numtiles; ++tileno,
	  ++tile) {
		/* initialize all tiles with JPC_TILE_DONE so
		   jpc_dec_destroy() knows which ones need a
		   jpc_dec_tilefini() call; they are not actually
		   "done", of course */
		tile->state = JPC_TILE_DONE;
	}

	for (tileno = 0, tile = dec->tiles; tileno < dec->numtiles; ++tileno,
	  ++tile) {
		htileno = tileno % dec->numhtiles;
		vtileno = tileno / dec->numhtiles;
		tile->realmode = 0;
		tile->state = JPC_TILE_INIT;
		tile->xstart = JAS_MAX(dec->tilexoff + htileno * dec->tilewidth,
		  dec->xstart);
		tile->ystart = JAS_MAX(dec->tileyoff + vtileno * dec->tileheight,
		  dec->ystart);
		tile->xend = JAS_MIN(dec->tilexoff + (htileno + 1) *
		  dec->tilewidth, dec->xend);
		tile->yend = JAS_MIN(dec->tileyoff + (vtileno + 1) *
		  dec->tileheight, dec->yend);
		tile->numparts = 0;
		tile->partno = 0;
		tile->pkthdrstream = 0;
		tile->pkthdrstreampos = 0;
		tile->pptstab = 0;
		tile->cp = 0;
		tile->pi = 0;
		if (!(tile->tcomps = jas_alloc2(dec->numcomps,
		  sizeof(jpc_dec_tcomp_t)))) {
			return -1;
		}
		for (compno = 0, cmpt = dec->cmpts, tcomp = tile->tcomps;
		  compno < dec->numcomps; ++compno, ++cmpt, ++tcomp) {
			tcomp->rlvls = 0;
			tcomp->numrlvls = 0;
			tcomp->data = 0;
			tcomp->xstart = JPC_CEILDIV(tile->xstart, cmpt->hstep);
			tcomp->ystart = JPC_CEILDIV(tile->ystart, cmpt->vstep);
			tcomp->xend = JPC_CEILDIV(tile->xend, cmpt->hstep);
			tcomp->yend = JPC_CEILDIV(tile->yend, cmpt->vstep);
			tcomp->tsfb = 0;
		}
	}

	dec->pkthdrstreams = 0;

	/* We should expect to encounter other main header marker segments
	  or an SOT marker segment next. */
	dec->state = JPC_MH;

	return 0;
}

Project: jasper
File: jpc_dec.c

static int jpc_dec_dump(jpc_dec_t *dec, FILE *out)
{
	jpc_dec_tile_t *tile;
	int tileno;
	jpc_dec_tcomp_t *tcomp;
	int compno;
	jpc_dec_rlvl_t *rlvl;
	unsigned rlvlno;
	jpc_dec_band_t *band;
	int bandno;
	jpc_dec_prc_t *prc;
	int prcno;
	jpc_dec_cblk_t *cblk;
	int cblkno;

	assert(!dec->numtiles || dec->tiles);
	for (tileno = 0, tile = dec->tiles; tileno < dec->numtiles;
	  ++tileno, ++tile) {
		assert(!dec->numcomps || tile->tcomps);
		for (compno = 0, tcomp = tile->tcomps; compno < dec->numcomps;
		  ++compno, ++tcomp) {
			for (rlvlno = 0, rlvl = tcomp->rlvls; rlvlno <
			  tcomp->numrlvls; ++rlvlno, ++rlvl) {
				fprintf(out, "RESOLUTION LEVEL %d\n", rlvlno);
				fprintf(out, "xs = %"PRIuFAST32", ys = %"PRIuFAST32", xe = %"PRIuFAST32", ye = %"PRIuFAST32", w = %"PRIuFAST32", h = %"PRIuFAST32"\n",
				  rlvl->xstart, rlvl->ystart, rlvl->xend, rlvl->yend,
				  rlvl->xend - rlvl->xstart, rlvl->yend - rlvl->ystart);
				assert(!rlvl->numbands || rlvl->bands);
				for (bandno = 0, band = rlvl->bands;
				  bandno < rlvl->numbands; ++bandno, ++band) {
					fprintf(out, "BAND %d\n", bandno);
					if (!band->data) {
						fprintf(out, "band has no data (null pointer)\n");
						assert(!band->prcs);
						continue;
					}
					fprintf(out, "xs = %"PRIiFAST32", ys = %"PRIiFAST32", xe = %"PRIiFAST32", ye = %"PRIiFAST32", w = %"PRIiFAST32", h = %"PRIiFAST32"\n",
					  jas_seq2d_xstart(band->data),
					  jas_seq2d_ystart(band->data),
					  jas_seq2d_xend(band->data),
					  jas_seq2d_yend(band->data),
					  jas_seq2d_xend(band->data) -
					  jas_seq2d_xstart(band->data),
					  jas_seq2d_yend(band->data) -
					  jas_seq2d_ystart(band->data));
					assert(!rlvl->numprcs || band->prcs);
					for (prcno = 0, prc = band->prcs;
					  prcno < rlvl->numprcs; ++prcno,
					  ++prc) {
						fprintf(out, "CODE BLOCK GROUP %d\n", prcno);
						fprintf(out, "xs = %"PRIuFAST32", ys = %"PRIuFAST32", xe = %"PRIuFAST32", ye = %"PRIuFAST32", w = %"PRIuFAST32", h = %"PRIuFAST32"\n",
						  prc->xstart, prc->ystart, prc->xend, prc->yend,
						  prc->xend - prc->xstart, prc->yend - prc->ystart);
						assert(!prc->numcblks || prc->cblks);
						for (cblkno = 0, cblk =
						  prc->cblks; cblkno <
						  prc->numcblks; ++cblkno,
						  ++cblk) {
							fprintf(out, "CODE BLOCK %d\n", cblkno);
							fprintf(out, "xs = %"PRIiFAST32", ys = %"PRIiFAST32", xe = %"PRIiFAST32", ye = %"PRIiFAST32", w = %"PRIiFAST32", h = %"PRIiFAST32"\n",
							  jas_seq2d_xstart(cblk->data),
							  jas_seq2d_ystart(cblk->data),
							  jas_seq2d_xend(cblk->data),
							  jas_seq2d_yend(cblk->data),
							  jas_seq2d_xend(cblk->data) -
							  jas_seq2d_xstart(cblk->data),
							  jas_seq2d_yend(cblk->data) -
							  jas_seq2d_ystart(cblk->data));
						}
					}
				}
			}
		}
	}

	return 0;
}