Commit message: jpc_cs: validate the component bit depth

According to the specification A.5.1 Table A-11, the component bit
depth can be at most 38.

Fixes CVE-2017-5499 (integer overflow)

Closes https://github.com/jasper-maint/jasper/issues/2
Closes https://github.com/mdadams/jasper/issues/63
CVE: CVE-2017-5499
CWE: CWE-190
Vulnerability description: Integer overflow in libjasper/jpc/jpc_dec.c in JasPer 1.900.17 allows remote attackers to cause a denial of service (crash) via a crafted file.

Project: jasper
File: jpc_cs.c

static int jpc_siz_getparms(jpc_ms_t *ms, jpc_cstate_t *cstate,
  jas_stream_t *in)
{
	jpc_siz_t *siz = &ms->parms.siz;
	unsigned int i;
	uint_fast8_t tmp;

	siz->comps = 0;

	/* Eliminate compiler warning about unused variables. */
	cstate = 0;

	if (jpc_getuint16(in, &siz->caps) ||
	  jpc_getuint32(in, &siz->width) ||
	  jpc_getuint32(in, &siz->height) ||
	  jpc_getuint32(in, &siz->xoff) ||
	  jpc_getuint32(in, &siz->yoff) ||
	  jpc_getuint32(in, &siz->tilewidth) ||
	  jpc_getuint32(in, &siz->tileheight) ||
	  jpc_getuint32(in, &siz->tilexoff) ||
	  jpc_getuint32(in, &siz->tileyoff) ||
	  jpc_getuint16(in, &siz->numcomps)) {
		goto error;
	}
	if (!siz->width || !siz->height) {
		jas_eprintf("reference grid cannot have zero area\n");
		goto error;
	}
	if (!siz->tilewidth || !siz->tileheight) {
		jas_eprintf("tile cannot have zero area\n");
		goto error;
	}
	if (!siz->numcomps || siz->numcomps > 16384) {
		jas_eprintf("number of components not in permissible range\n");
		goto error;
	}
	if (siz->xoff >= siz->width) {
		jas_eprintf("XOsiz not in permissible range\n");
		goto error;
	}
	if (siz->yoff >= siz->height) {
		jas_eprintf("YOsiz not in permissible range\n");
		goto error;
	}
	if (siz->tilexoff > siz->xoff || siz->tilexoff + siz->tilewidth <= siz->xoff) {
		jas_eprintf("XTOsiz not in permissible range\n");
		goto error;
	}
	if (siz->tileyoff > siz->yoff || siz->tileyoff + siz->tileheight <= siz->yoff) {
		jas_eprintf("YTOsiz not in permissible range\n");
		goto error;
	}

	if (!(siz->comps = jas_alloc2(siz->numcomps, sizeof(jpc_sizcomp_t)))) {
		goto error;
	}
	for (i = 0; i < siz->numcomps; ++i) {
		if (jpc_getuint8(in, &tmp) ||
		  jpc_getuint8(in, &siz->comps[i].hsamp) ||
		  jpc_getuint8(in, &siz->comps[i].vsamp)) {
			goto error;
		}
		if (siz->comps[i].hsamp == 0 || siz->comps[i].hsamp > 255) {
			jas_eprintf("invalid XRsiz value %d\n", siz->comps[i].hsamp);
			goto error;
		}
		if (siz->comps[i].vsamp == 0 || siz->comps[i].vsamp > 255) {
			jas_eprintf("invalid YRsiz value %d\n", siz->comps[i].vsamp);
			goto error;
		}
		siz->comps[i].sgnd = (tmp >> 7) & 1;
		siz->comps[i].prec = (tmp & 0x7f) + 1;
		if (siz->comps[i].prec > 38) {
			jas_eprintf("invalid component bit depth %d\n", siz->comps[i].prec);
			goto error;
		}
	}
	if (jas_stream_eof(in)) {
		goto error;
	}
	return 0;

error:
	if (siz->comps) {
		jas_free(siz->comps);
	}
	return -1;
}

Project: jasper
File: jpc_cs.c

static const jpc_mstabent_t *jpc_mstab_lookup(int id)
{
	const jpc_mstabent_t *mstabent;
	for (mstabent = jpc_mstab;; ++mstabent) {
		if (mstabent->id == id || mstabent->id < 0) {
			return mstabent;
		}
	}
	assert(0);
	return 0;
}

Project: jasper
File: jpc_cs.c

jpc_ms_t *jpc_ms_create(int type)
{
	jpc_ms_t *ms;
	const jpc_mstabent_t *mstabent;

	if (!(ms = jas_malloc(sizeof(jpc_ms_t)))) {
		return 0;
	}
	ms->id = type;
	ms->len = 0;
	mstabent = jpc_mstab_lookup(ms->id);
	ms->ops = &mstabent->ops;
	memset(&ms->parms, 0, sizeof(jpc_msparms_t));
	return ms;
}

Project: jasper
File: jpc_cs.c

jpc_ms_t *jpc_getms(jas_stream_t *in, jpc_cstate_t *cstate)
{
	jpc_ms_t *ms;
	const jpc_mstabent_t *mstabent;
	jas_stream_t *tmpstream;

	if (!(ms = jpc_ms_create(0))) {
		return 0;
	}

	/* Get the marker type. */
	if (jpc_getuint16(in, &ms->id) || ms->id < JPC_MS_MIN ||
	  ms->id > JPC_MS_MAX) {
		jpc_ms_destroy(ms);
		return 0;
	}

	mstabent = jpc_mstab_lookup(ms->id);
	ms->ops = &mstabent->ops;

	/* Get the marker segment length and parameters if present. */
	/* Note: It is tacitly assumed that a marker segment cannot have
	  parameters unless it has a length field.  That is, there cannot
	  be a parameters field without a length field and vice versa. */
	if (JPC_MS_HASPARMS(ms->id)) {
		/* Get the length of the marker segment. */
		if (jpc_getuint16(in, &ms->len) || ms->len < 3) {
			jpc_ms_destroy(ms);
			return 0;
		}
		/* Calculate the length of the marker segment parameters. */
		ms->len -= 2;
		/* Create and prepare a temporary memory stream from which to
		  read the marker segment parameters. */
		/* Note: This approach provides a simple way of ensuring that
		  we never read beyond the end of the marker segment (even if
		  the marker segment length is errantly set too small). */
		if (!(tmpstream = jas_stream_memopen(0, 0))) {
			jpc_ms_destroy(ms);
			return 0;
		}
		if (jas_stream_copy(tmpstream, in, ms->len) ||
		  jas_stream_seek(tmpstream, 0, SEEK_SET) < 0) {
			jas_stream_close(tmpstream);
			jpc_ms_destroy(ms);
			return 0;
		}
		/* Get the marker segment parameters. */
		if ((*ms->ops->getparms)(ms, cstate, tmpstream)) {
			ms->ops = 0;
			jpc_ms_destroy(ms);
			jas_stream_close(tmpstream);
			return 0;
		}

		if (jas_getdbglevel() > 0) {
			jpc_ms_dump(ms, stderr);
		}

		if (JAS_CAST(jas_ulong, jas_stream_tell(tmpstream)) != ms->len) {
			jas_eprintf(
			  "warning: trailing garbage in marker segment (%ld bytes)\n",
			  ms->len - jas_stream_tell(tmpstream));
		}

		/* Close the temporary stream. */
		jas_stream_close(tmpstream);

	} else {
		/* There are no marker segment parameters. */
		ms->len = 0;

		if (jas_getdbglevel() > 0) {
			jpc_ms_dump(ms, stderr);
		}
	}

	/* Update the code stream state information based on the type of
	  marker segment read. */
	/* Note: This is a bit of a hack, but I'm not going to define another
	  type of virtual function for this one special case. */
	if (ms->id == JPC_MS_SIZ) {
		cstate->numcomps = ms->parms.siz.numcomps;
	}

	return ms;
}

Project: jasper
File: jpc_cs.c

void jpc_ms_dump(jpc_ms_t *ms, FILE *out)
{
	const jpc_mstabent_t *mstabent;
	mstabent = jpc_mstab_lookup(ms->id);
	fprintf(out, "type = 0x%04"PRIxFAST16" (%s);", ms->id, mstabent->name);
	if (JPC_MS_HASPARMS(ms->id)) {
		fprintf(out, " len = %"PRIuFAST16";", ms->len + 2);
		if (ms->ops->dumpparms) {
			(*ms->ops->dumpparms)(ms, out);
		} else {
			fprintf(out, "\n");
		}
	} else {
		fprintf(out, "\n");
	}
}