Commit message: Added some additional checking to prevent a potential integer overflow
due to conversion in the JPC decoder.
CVE: CVE-2016-9387
CWE: CWE-190
Vulnerability description: Integer overflow in the jpc_dec_process_siz function in libjasper/jpc/jpc_dec.c in JasPer before 1.900.13 allows remote attackers to have unspecified impact via a crafted file, which triggers an assertion failure.

Project: jasper
File: jpc_dec.c

static void calcstepsizes(uint_fast16_t refstepsize, int numrlvls,
  uint_fast16_t *stepsizes)
{
	int bandno;
	int numbands;
	uint_fast16_t expn;
	uint_fast16_t mant;
	expn = JPC_QCX_GETEXPN(refstepsize);
	mant = JPC_QCX_GETMANT(refstepsize);
	numbands = 3 * numrlvls - 2;
	for (bandno = 0; bandno < numbands; ++bandno) {
//jas_eprintf("DEBUG %d %d %d %d %d\n", bandno, expn, numrlvls, bandno, ((numrlvls - 1) - (numrlvls - 1 - ((bandno > 0) ? ((bandno + 2) / 3) : (0)))));
		stepsizes[bandno] = JPC_QCX_MANT(mant) | JPC_QCX_EXPN(expn +
		  (numrlvls - 1) - (numrlvls - 1 - ((bandno > 0) ? ((bandno + 2) / 3) : (0))));
	}
}

Project: jasper
File: jpc_dec.c

static int jpc_dec_tilefini(jpc_dec_t *dec, jpc_dec_tile_t *tile)
{
	jpc_dec_tcomp_t *tcomp;
	int compno;
	int bandno;
	int rlvlno;
	jpc_dec_band_t *band;
	jpc_dec_rlvl_t *rlvl;
	int prcno;
	jpc_dec_prc_t *prc;
	jpc_dec_seg_t *seg;
	jpc_dec_cblk_t *cblk;
	int cblkno;

	if (tile->tcomps) {

		for (compno = 0, tcomp = tile->tcomps; compno < dec->numcomps;
		  ++compno, ++tcomp) {
			for (rlvlno = 0, rlvl = tcomp->rlvls; rlvlno < tcomp->numrlvls;
			  ++rlvlno, ++rlvl) {
				if (!rlvl->bands) {
					continue;
				}
				for (bandno = 0, band = rlvl->bands; bandno < rlvl->numbands;
				  ++bandno, ++band) {
					if (band->prcs) {
						for (prcno = 0, prc = band->prcs; prcno <
						  rlvl->numprcs; ++prcno, ++prc) {
							if (!prc->cblks) {
								continue;
							}
							for (cblkno = 0, cblk = prc->cblks; cblkno <
							  prc->numcblks; ++cblkno, ++cblk) {

								while (cblk->segs.head) {
									seg = cblk->segs.head;
									jpc_seglist_remove(&cblk->segs, seg);
									jpc_seg_destroy(seg);
								}
								jas_matrix_destroy(cblk->data);
								if (cblk->mqdec) {
									jpc_mqdec_destroy(cblk->mqdec);
								}
								if (cblk->nulldec) {
									jpc_bitstream_close(cblk->nulldec);
								}
								if (cblk->flags) {
									jas_matrix_destroy(cblk->flags);
								}
							}
							if (prc->incltagtree) {
								jpc_tagtree_destroy(prc->incltagtree);
							}
							if (prc->numimsbstagtree) {
								jpc_tagtree_destroy(prc->numimsbstagtree);
							}
							if (prc->cblks) {
								jas_free(prc->cblks);
							}
						}
					}
					if (band->data) {
						jas_matrix_destroy(band->data);
					}
					if (band->prcs) {
						jas_free(band->prcs);
					}
				}
				if (rlvl->bands) {
					jas_free(rlvl->bands);
				}
			}
			if (tcomp->rlvls) {
				jas_free(tcomp->rlvls);
			}
			if (tcomp->data) {
				jas_matrix_destroy(tcomp->data);
			}
			if (tcomp->tsfb) {
				jpc_tsfb_destroy(tcomp->tsfb);
			}
		}
	}

	if (tile->cp) {
		jpc_dec_cp_destroy(tile->cp);
		//tile->cp = 0;
	}
	if (tile->tcomps) {
		jas_free(tile->tcomps);
		//tile->tcomps = 0;
	}
	if (tile->pi) {
		jpc_pi_destroy(tile->pi);
		//tile->pi = 0;
	}
	if (tile->pkthdrstream) {
		jas_stream_close(tile->pkthdrstream);
		//tile->pkthdrstream = 0;
	}
	if (tile->pptstab) {
		jpc_ppxstab_destroy(tile->pptstab);
		//tile->pptstab = 0;
	}

	tile->state = JPC_TILE_DONE;

	return 0;
}

Project: jasper
File: jpc_dec.c

int jpc_pptstabwrite(jas_stream_t *out, jpc_ppxstab_t *tab)
{
	int i;
	jpc_ppxstabent_t *ent;
	for (i = 0; i < tab->numents; ++i) {
		ent = tab->ents[i];
		if (jas_stream_write(out, ent->data, ent->len) != JAS_CAST(int, ent->len)) {
			return -1;
		}
	}
	return 0;
}

Project: jasper
File: jpc_dec.c

static int jpc_dec_process_eoc(jpc_dec_t *dec, jpc_ms_t *ms)
{
	int tileno;
	jpc_dec_tile_t *tile;

	/* Eliminate compiler warnings about unused variables. */
	ms = 0;

	for (tileno = 0, tile = dec->tiles; tileno < dec->numtiles; ++tileno,
	  ++tile) {
		if (tile->state == JPC_TILE_ACTIVE) {
			if (jpc_dec_tiledecode(dec, tile)) {
				return -1;
			}
		}
		/* If the tile has not yet been finalized, finalize it. */
		// OLD CODE: jpc_dec_tilefini(dec, tile);
		if (tile->state != JPC_TILE_DONE) {
			jpc_dec_tilefini(dec, tile);
		}
	}

	/* We are done processing the code stream. */
	dec->state = JPC_MT;

	return 1;
}