Commit message: Changed the JPC bitstream code to more gracefully handle a request
for a larger sized integer than what can be handled (i.e., return
with an error instead of failing an assert).
CVE: CVE-2016-9391
CWE: CWE-703
Vulnerability description: The jpc_bitstream_getbits function in jpc_bs.c in JasPer before 2.0.10 allows remote attackers to cause a denial of service (assertion failure) via a very large integer.

Project: jasper
File: jpc_bs.c

int jpc_bitstream_align(jpc_bitstream_t *bitstream)
{
	int ret;
	if (bitstream->openmode_ & JPC_BITSTREAM_READ) {
		ret = jpc_bitstream_inalign(bitstream, 0, 0);
	} else if (bitstream->openmode_ & JPC_BITSTREAM_WRITE) {
		ret = jpc_bitstream_outalign(bitstream, 0);
	} else {
		abort();
	}
	return ret;
}

Project: jasper
File: jpc_bs.c

static jpc_bitstream_t *jpc_bitstream_alloc()
{
	jpc_bitstream_t *bitstream;

	/* Allocate memory for the new bit stream object. */
	if (!(bitstream = jas_malloc(sizeof(jpc_bitstream_t)))) {
		return 0;
	}
	/* Initialize all of the data members. */
	bitstream->stream_ = 0;
	bitstream->cnt_ = 0;
	bitstream->flags_ = 0;
	bitstream->openmode_ = 0;

	return bitstream;
}

Project: jasper
File: jpc_bs.c

int jpc_bitstream_close(jpc_bitstream_t *bitstream)
{
	int ret = 0;

	/* Align to the next byte boundary while considering the effects of
	  bit stuffing. */
	if (jpc_bitstream_align(bitstream)) {
		ret = -1;
	}

	/* If necessary, close the underlying (character) stream. */
	if (!(bitstream->flags_ & JPC_BITSTREAM_NOCLOSE) && bitstream->stream_) {
		if (jas_stream_close(bitstream->stream_)) {
			ret = -1;
		}
		bitstream->stream_ = 0;
	}

	jas_free(bitstream);
	return ret;
}

Project: jasper
File: jpc_bs.c

int jpc_bitstream_fillbuf(jpc_bitstream_t *bitstream)
{
	int c;
	/* Note: The count has already been decremented by the caller. */
	assert(bitstream->openmode_ & JPC_BITSTREAM_READ);
	assert(bitstream->cnt_ <= 0);

	if (bitstream->flags_ & JPC_BITSTREAM_ERR) {
		bitstream->cnt_ = 0;
		return -1;
	}

	if (bitstream->flags_ & JPC_BITSTREAM_EOF) {
		bitstream->buf_ = 0x7f;
		bitstream->cnt_ = 7;
		return 1;
	}

	bitstream->buf_ = (bitstream->buf_ << 8) & 0xffff;
	if ((c = jas_stream_getc((bitstream)->stream_)) == EOF) {
		bitstream->flags_ |= JPC_BITSTREAM_EOF;
		return 1;
	}
	bitstream->cnt_ = (bitstream->buf_ == 0xff00) ? 6 : 7;
	bitstream->buf_ |= c & ((1 << (bitstream->cnt_ + 1)) - 1);
	return (bitstream->buf_ >> bitstream->cnt_) & 1;
}

Project: jasper
File: jpc_bs.c

int jpc_bitstream_getbit_func(jpc_bitstream_t *bitstream)
{
	int ret;
	JAS_DBGLOG(1000, ("jpc_bitstream_getbit_func(%p)\n", bitstream));
	ret = jpc_bitstream_getbit_macro(bitstream);
	JAS_DBGLOG(1000, ("jpc_bitstream_getbit_func -> %d\n", ret));
	return ret;
}

Project: jasper
File: jpc_bs.c

int jpc_bitstream_inalign(jpc_bitstream_t *bitstream, int fillmask,
  int filldata)
{
	int n;
	int v;
	int u;
	int numfill;
	int m;

	numfill = 7;
	m = 0;
	v = 0;
	if (bitstream->cnt_ > 0) {
		n = bitstream->cnt_;
	} else if (!bitstream->cnt_) {
		n = ((bitstream->buf_ & 0xff) == 0xff) ? 7 : 0;
	} else {
		n = 0;
	}
	if (n > 0) {
		if ((u = jpc_bitstream_getbits(bitstream, n)) < 0) {
			return -1;
		}
		m += n;
		v = (v << n) | u;
	}
	if ((bitstream->buf_ & 0xff) == 0xff) {
		if ((u = jpc_bitstream_getbits(bitstream, 7)) < 0) {
			return -1;
		}
		v = (v << 7) | u;
		m += 7;
	}
	if (m > numfill) {
		v >>= m - numfill;
	} else {
		filldata >>= numfill - m;
		fillmask >>= numfill - m;
	}
	if (((~(v ^ filldata)) & fillmask) != fillmask) {
		/* The actual fill pattern does not match the expected one. */
		return 1;
	}

	return 0;
}

Project: jasper
File: jpc_bs.c

int jpc_bitstream_needalign(jpc_bitstream_t *bitstream)
{
	if (bitstream->openmode_ & JPC_BITSTREAM_READ) {
		/* The bit stream is open for reading. */
		/* If there are any bits buffered for reading, or the
		  previous byte forced a stuffed bit, alignment is
		  required. */
		if ((bitstream->cnt_ < 8 && bitstream->cnt_ > 0) ||
		  ((bitstream->buf_ >> 8) & 0xff) == 0xff) {
			return 1;
		}
	} else if (bitstream->openmode_ & JPC_BITSTREAM_WRITE) {
		/* The bit stream is open for writing. */
		/* If there are any bits buffered for writing, or the
		  previous byte forced a stuffed bit, alignment is
		  required. */
		if ((bitstream->cnt_ < 8 && bitstream->cnt_ >= 0) ||
		  ((bitstream->buf_ >> 8) & 0xff) == 0xff) {
			return 1;
		}
	} else {
		/* This should not happen.  Famous last words, eh? :-) */
		assert(0);
		return -1;
	}
	return 0;
}

Project: jasper
File: jpc_bs.c

int jpc_bitstream_outalign(jpc_bitstream_t *bitstream, int filldata)
{
	int n;
	int v;

	/* Ensure that this bit stream is open for writing. */
	assert(bitstream->openmode_ & JPC_BITSTREAM_WRITE);

	/* Ensure that the first bit of fill data is zero. */
	/* Note: The first bit of fill data must be zero.  If this were not
	  the case, the fill data itself could cause further bit stuffing to
	  be required (which would cause numerous complications). */
	assert(!(filldata & (~0x3f)));

	if (!bitstream->cnt_) {
		if ((bitstream->buf_ & 0xff) == 0xff) {
			n = 7;
			v = filldata;
		} else {
			n = 0;
			v = 0;
		}
	} else if (bitstream->cnt_ > 0 && bitstream->cnt_ < 8) {
		n = bitstream->cnt_;
		v = filldata >> (7 - n);
	} else {
		n = 0;
		v = 0;
		return 0;
	}

	/* Write the appropriate fill data to the bit stream. */
	if (n > 0) {
		if (jpc_bitstream_putbits(bitstream, n, v)) {
			return -1;
		}
	}
	if (bitstream->cnt_ < 8) {
		assert(bitstream->cnt_ >= 0 && bitstream->cnt_ < 8);
		assert((bitstream->buf_ & 0xff) != 0xff);
		/* Force the pending byte of output to be written to the
		  underlying (character) stream. */
		if (jas_stream_putc(bitstream->stream_, bitstream->buf_ & 0xff) == EOF) {
			return -1;
		}
		bitstream->cnt_ = 8;
		bitstream->buf_ = (bitstream->buf_ << 8) & 0xffff;
	}

	return 0;
}

Project: jasper
File: jpc_bs.c

int jpc_bitstream_pending(jpc_bitstream_t *bitstream)
{
	if (bitstream->openmode_ & JPC_BITSTREAM_WRITE) {
		/* The bit stream is being used for writing. */
#if 1
		/* XXX - Is this really correct?  Check someday... */
		if (bitstream->cnt_ < 8) {
			return 1;
		}
#else
		if (bitstream->cnt_ < 8) {
			if (((bitstream->buf_ >> 8) & 0xff) == 0xff) {
				return 2;
			}
			return 1;
		}
#endif
		return 0;
	} else {
		/* This operation should not be invoked on a bit stream that
		  is being used for reading. */
		return -1;
	}
}

Project: jasper
File: jpc_bs.c

jpc_bitstream_t *jpc_bitstream_sopen(jas_stream_t *stream, char *mode)
{
	jpc_bitstream_t *bitstream;

	/* Ensure that the open mode is valid. */
#if 0 /* This causes a string literal too long error (with c99 pedantic mode).  Why is this so? */
	assert(!strcmp(mode, "r") || !strcmp(mode, "w") || !strcmp(mode, "r+")
	  || !strcmp(mode, "w+"));
#endif

	if (!(bitstream = jpc_bitstream_alloc())) {
		return 0;
	}

	/* By default, do not close the underlying (character) stream, upon
	  the close of the bit stream. */
	bitstream->flags_ = JPC_BITSTREAM_NOCLOSE;

	bitstream->stream_ = stream;
	bitstream->openmode_ = (mode[0] == 'w') ? JPC_BITSTREAM_WRITE :
	  JPC_BITSTREAM_READ;

	/* Mark the data buffer as empty. */
	bitstream->cnt_ = (bitstream->openmode_ == JPC_BITSTREAM_READ) ? 0 : 8;
	bitstream->buf_ = 0;

	return bitstream;
}

Project: jasper
File: jpc_dec.c

static int jpc_dec_dump(jpc_dec_t *dec, FILE *out)
{
	jpc_dec_tile_t *tile;
	int tileno;
	jpc_dec_tcomp_t *tcomp;
	int compno;
	jpc_dec_rlvl_t *rlvl;
	int rlvlno;
	jpc_dec_band_t *band;
	int bandno;
	jpc_dec_prc_t *prc;
	int prcno;
	jpc_dec_cblk_t *cblk;
	int cblkno;

	for (tileno = 0, tile = dec->tiles; tileno < dec->numtiles;
	  ++tileno, ++tile) {
		for (compno = 0, tcomp = tile->tcomps; compno < dec->numcomps;
		  ++compno, ++tcomp) {
			for (rlvlno = 0, rlvl = tcomp->rlvls; rlvlno <
			  tcomp->numrlvls; ++rlvlno, ++rlvl) {
				fprintf(out, "RESOLUTION LEVEL %d\n", rlvlno);
				fprintf(out, "xs =%"PRIuFAST32", ys = %"PRIuFAST32", xe = %"PRIuFAST32", ye = %"PRIuFAST32", w = %"PRIuFAST32", h = %"PRIuFAST32"\n",
				  rlvl->xstart, rlvl->ystart, rlvl->xend, rlvl->yend, rlvl->xend -
				  rlvl->xstart, rlvl->yend - rlvl->ystart);
				for (bandno = 0, band = rlvl->bands;
				  bandno < rlvl->numbands; ++bandno, ++band) {
					fprintf(out, "BAND %d\n", bandno);
					fprintf(out, "xs =%"PRIiFAST32", ys = %"PRIiFAST32", xe = %"PRIiFAST32", ye = %"PRIiFAST32", w = %"PRIiFAST32", h = %"PRIiFAST32"\n",
					  jas_seq2d_xstart(band->data), jas_seq2d_ystart(band->data), jas_seq2d_xend(band->data),
					  jas_seq2d_yend(band->data), jas_seq2d_xend(band->data) - jas_seq2d_xstart(band->data),
					  jas_seq2d_yend(band->data) - jas_seq2d_ystart(band->data));
					for (prcno = 0, prc = band->prcs;
					  prcno < rlvl->numprcs; ++prcno,
					  ++prc) {
						fprintf(out, "CODE BLOCK GROUP %d\n", prcno);
						fprintf(out, "xs =%"PRIuFAST32", ys = %"PRIuFAST32", xe = %"PRIuFAST32", ye = %"PRIuFAST32", w = %"PRIuFAST32", h = %"PRIuFAST32"\n",
						  prc->xstart, prc->ystart, prc->xend, prc->yend, prc->xend -
						  prc->xstart, prc->yend - prc->ystart);
						for (cblkno = 0, cblk =
						  prc->cblks; cblkno <
						  prc->numcblks; ++cblkno,
						  ++cblk) {
							fprintf(out, "CODE BLOCK %d\n", cblkno);
							fprintf(out, "xs =%"PRIiFAST32", ys = %"PRIiFAST32", xe = %"PRIiFAST32", ye = %"PRIiFAST32", w = %"PRIiFAST32", h = %"PRIiFAST32"\n",
							  jas_seq2d_xstart(cblk->data), jas_seq2d_ystart(cblk->data), jas_seq2d_xend(cblk->data),
							  jas_seq2d_yend(cblk->data), jas_seq2d_xend(cblk->data) - jas_seq2d_xstart(cblk->data),
							  jas_seq2d_yend(cblk->data) - jas_seq2d_ystart(cblk->data));
						}
					}
				}
			}
		}
	}

	return 0;
}