Commit message: Check data in jas_image

Regarding CVE-2018-19539.
Fix by Markus Koschany <apo@debian.org>.
From https://gist.github.com/apoleon/7c0f3a0c28437c18fee8a51b1aa16164.
CVE: CVE-2018-19539
CWE: CWE-617
Vulnerability description: An issue was discovered in JasPer 2.0.14. There is an access violation in the function jas_image_readcmpt in libjasper/base/jas_image.c, leading to a denial of service.

Project: jasper
File: jas_image.c

static inline jas_ulong encode_twos_comp(long n, int prec)
{
	jas_ulong result;
	assert(prec >= 2);
	jas_eprintf("warning: support for signed data is untested\n");
	// NOTE: Is this correct?
	if (n < 0) {
		result = -n;
		result = (result ^ 0xffffffffUL) + 1;
		result &= (1 << prec) - 1;
	} else {
		result = n;
	}
	return result;
}

Project: jasper
File: jas_image.c

jas_image_t *jas_image_create0()
{
	jas_image_t *image;

	if (!(image = jas_malloc(sizeof(jas_image_t)))) {
		return 0;
	}

	image->tlx_ = 0;
	image->tly_ = 0;
	image->brx_ = 0;
	image->bry_ = 0;
	image->clrspc_ = JAS_CLRSPC_UNKNOWN;
	image->numcmpts_ = 0;
	image->maxcmpts_ = 0;
	image->cmpts_ = 0;
//	image->inmem_ = true;
	image->cmprof_ = 0;

	return image;
}

Project: jasper
File: jas_image.c

int jas_image_encode(jas_image_t *image, jas_stream_t *out, int fmt,
  const char *optstr)
{
	jas_image_fmtinfo_t *fmtinfo;
	if (!(fmtinfo = jas_image_lookupfmtbyid(fmt))) {
		jas_eprintf("format lookup failed\n");
		return -1;
	}
	return (fmtinfo->ops.encode) ? (*fmtinfo->ops.encode)(image, out,
	  optstr) : (-1);
}

Project: jasper
File: jas_image.c

jas_image_t *jas_image_create(int numcmpts, jas_image_cmptparm_t *cmptparms,
  int clrspc)
{
	jas_image_t *image;
	size_t rawsize;
	uint_fast32_t inmem;
	int cmptno;
	jas_image_cmptparm_t *cmptparm;

	image = 0;

	JAS_DBGLOG(100, ("jas_image_create(%d, %p, %d)\n", numcmpts, cmptparms,
	  clrspc));

	if (!(image = jas_image_create0())) {
		goto error;
	}

	image->clrspc_ = clrspc;
	image->maxcmpts_ = numcmpts;
//	image->inmem_ = true;

	/* Allocate memory for the per-component information. */
	if (!(image->cmpts_ = jas_alloc2(image->maxcmpts_,
	  sizeof(jas_image_cmpt_t *)))) {
		goto error;
	}
	/* Initialize in case of failure. */
	for (cmptno = 0; cmptno < image->maxcmpts_; ++cmptno) {
		image->cmpts_[cmptno] = 0;
	}

#if 0
	/* Compute the approximate raw size of the image. */
	rawsize = 0;
	for (cmptno = 0, cmptparm = cmptparms; cmptno < numcmpts; ++cmptno,
	  ++cmptparm) {
		rawsize += cmptparm->width * cmptparm->height *
		  (cmptparm->prec + 7) / 8;
	}
	/* Decide whether to buffer the image data in memory, based on the
	  raw size of the image. */
	inmem = (rawsize < JAS_IMAGE_INMEMTHRESH);
#endif

	/* Create the individual image components. */
	for (cmptno = 0, cmptparm = cmptparms; cmptno < numcmpts; ++cmptno,
	  ++cmptparm) {
		if (!jas_safe_size_mul3(cmptparm->width, cmptparm->height,
		  (cmptparm->prec + 7), &rawsize)) {
			goto error;
		}
		rawsize /= 8;
		inmem = (rawsize < JAS_IMAGE_INMEMTHRESH);
		if (!(image->cmpts_[cmptno] = jas_image_cmpt_create(cmptparm->tlx,
		  cmptparm->tly, cmptparm->hstep, cmptparm->vstep,
		  cmptparm->width, cmptparm->height, cmptparm->prec,
		  cmptparm->sgnd, inmem))) {
			goto error;
		}
		++image->numcmpts_;
	}

	/* Determine the bounding box for all of the components on the
	  reference grid (i.e., the image area) */
	jas_image_setbbox(image);

	return image;

error:
	if (image) {
		jas_image_destroy(image);
	}
	return 0;
}

Project: jasper
File: jas_image.c

static inline long decode_twos_comp(jas_ulong c, int prec)
{
	long result;
	assert(prec >= 2);
	jas_eprintf("warning: support for signed data is untested\n");
	// NOTE: Is this correct?
	result = (c & ((1 << (prec - 1)) - 1)) - (c & (1 << (prec - 1)));
	return result;
}

Project: jasper
File: jas_image.c

static jas_image_cmpt_t *jas_image_cmpt_create(int_fast32_t tlx,
  int_fast32_t tly, int_fast32_t hstep, int_fast32_t vstep,
  int_fast32_t width, int_fast32_t height, uint_fast16_t depth, bool sgnd,
  uint_fast32_t inmem)
{
	jas_image_cmpt_t *cmpt;
	size_t size;

	JAS_DBGLOG(100, (
	  "jas_image_cmpt_create(%ld, %ld, %ld, %ld, %ld, %ld, %d, %d, %d)\n",
	  JAS_CAST(long, tlx),
	  JAS_CAST(long, tly),
	  JAS_CAST(long, hstep),
	  JAS_CAST(long, vstep),
	  JAS_CAST(long, width),
	  JAS_CAST(long, height),
	  JAS_CAST(int, depth),
	  sgnd,
	  inmem
	  ));

	cmpt = 0;
	if (width < 0 || height < 0 || hstep <= 0 || vstep <= 0) {
		goto error;
	}
	if (!jas_safe_intfast32_add(tlx, width, 0) ||
	  !jas_safe_intfast32_add(tly, height, 0)) {
		goto error;
	}
	if (!jas_safe_intfast32_mul3(width, height, depth, 0)) {
		goto error;
	}

	if (!(cmpt = jas_malloc(sizeof(jas_image_cmpt_t)))) {
		goto error;
	}

	cmpt->type_ = JAS_IMAGE_CT_UNKNOWN;
	cmpt->tlx_ = tlx;
	cmpt->tly_ = tly;
	cmpt->hstep_ = hstep;
	cmpt->vstep_ = vstep;
	cmpt->width_ = width;
	cmpt->height_ = height;
	cmpt->prec_ = depth;
	cmpt->sgnd_ = sgnd;
	cmpt->stream_ = 0;
	cmpt->cps_ = (depth + 7) / 8;

	// Compute the number of samples in the image component, while protecting
	// against overflow.
	// size = cmpt->width_ * cmpt->height_ * cmpt->cps_;
	if (!jas_safe_size_mul3(cmpt->width_, cmpt->height_, cmpt->cps_, &size)) {
		goto error;
	}
	cmpt->stream_ = (inmem) ? jas_stream_memopen2(0, size) :
	  jas_stream_tmpfile();
	if (!cmpt->stream_) {
		goto error;
	}

	/* Zero the component data.  This isn't necessary, but it is
	convenient for debugging purposes. */
	/* Note: conversion of size - 1 to long can overflow */
	if (size > 0) {
		if (size - 1 > LONG_MAX) {
			goto error;
		}
		if (jas_stream_seek(cmpt->stream_, size - 1, SEEK_SET) < 0 ||
		  jas_stream_putc(cmpt->stream_, 0) == EOF ||
		  jas_stream_seek(cmpt->stream_, 0, SEEK_SET) < 0) {
			goto error;
		}
	}

	return cmpt;

error:
	if (cmpt) {
		jas_image_cmpt_destroy(cmpt);
	}
	return 0;
}

Project: jasper
File: jas_image.c

int jas_image_readcmpt(jas_image_t *image, int cmptno, jas_image_coord_t x,
  jas_image_coord_t y, jas_image_coord_t width, jas_image_coord_t height,
  jas_matrix_t *data)
{
	jas_image_cmpt_t *cmpt;
	jas_image_coord_t i;
	jas_image_coord_t j;
	int k;
	jas_seqent_t v;
	int c;
	jas_seqent_t *dr;
	jas_seqent_t *d;
	int drs;

	JAS_DBGLOG(100, ("jas_image_readcmpt(%p, %d, %ld, %ld, %ld, %ld, %p)\n",
	  image, cmptno, JAS_CAST(long, x), JAS_CAST(long, y),
	  JAS_CAST(long, width), JAS_CAST(long, height), data));

	if(data == NULL) {
		return -1;
	}

	if (cmptno < 0 || cmptno >= image->numcmpts_) {
		return -1;
	}

	cmpt = image->cmpts_[cmptno];
	if (x >= cmpt->width_ || y >= cmpt->height_ ||
	  x + width > cmpt->width_ ||
	  y + height > cmpt->height_) {
		return -1;
	}

	if (!jas_matrix_numrows(data) || !jas_matrix_numcols(data)) {
		return -1;
	}

	if (jas_matrix_numrows(data) != height || jas_matrix_numcols(data) != width) {
		if (jas_matrix_resize(data, height, width)) {
			return -1;
		}
	}

	dr = jas_matrix_getref(data, 0, 0);
	drs = jas_matrix_rowstep(data);
	for (i = 0; i < height; ++i, dr += drs) {
		d = dr;
		if (jas_stream_seek(cmpt->stream_, (cmpt->width_ * (y + i) + x)
		  * cmpt->cps_, SEEK_SET) < 0) {
			return -1;
		}
		for (j = width; j > 0; --j, ++d) {
			v = 0;
			for (k = cmpt->cps_; k > 0; --k) {
				if ((c = jas_stream_getc(cmpt->stream_)) == EOF) {
					return -1;
				}
				v = (v << 8) | (c & 0xff);
			}
			*d = bitstoint(v, cmpt->prec_, cmpt->sgnd_);
		}
	}

	return 0;
}

Project: jasper
File: jas_image.c

int jas_image_writecmpt(jas_image_t *image, int cmptno, jas_image_coord_t x,
  jas_image_coord_t y, jas_image_coord_t width, jas_image_coord_t height,
  jas_matrix_t *data)
{
	jas_image_cmpt_t *cmpt;
	jas_image_coord_t i;
	jas_image_coord_t j;
	jas_seqent_t *d;
	jas_seqent_t *dr;
	int drs;
	jas_seqent_t v;
	int k;
	int c;

	JAS_DBGLOG(100, ("jas_image_writecmpt(%p, %d, %ld, %ld, %ld, %ld, %p)\n",
	  image, cmptno, JAS_CAST(long, x), JAS_CAST(long, y),
	  JAS_CAST(long, width), JAS_CAST(long, height), data));

	if (cmptno < 0 || cmptno >= image->numcmpts_) {
		return -1;
	}

	cmpt = image->cmpts_[cmptno];
	if (x >= cmpt->width_ || y >= cmpt->height_ ||
	  x + width > cmpt->width_ ||
	  y + height > cmpt->height_) {
		return -1;
	}

	if (!jas_matrix_numrows(data) || !jas_matrix_numcols(data)) {
		return -1;
	}

	if (jas_matrix_numrows(data) != height ||
	  jas_matrix_numcols(data) != width) {
		return -1;
	}

	dr = jas_matrix_getref(data, 0, 0);
	drs = jas_matrix_rowstep(data);
	for (i = 0; i < height; ++i, dr += drs) {
		d = dr;
		if (jas_stream_seek(cmpt->stream_, (cmpt->width_ * (y + i) + x)
		  * cmpt->cps_, SEEK_SET) < 0) {
			return -1;
		}
		for (j = width; j > 0; --j, ++d) {
			v = inttobits(*d, cmpt->prec_, cmpt->sgnd_);
			for (k = cmpt->cps_; k > 0; --k) {
				c = (v >> (8 * (cmpt->cps_ - 1))) & 0xff;
				if (jas_stream_putc(cmpt->stream_,
				  (unsigned char) c) == EOF) {
					return -1;
				}
				v <<= 8;
			}
		}
	}

	return 0;
}

Project: jasper
File: jas_image.c

jas_image_t *jas_image_decode(jas_stream_t *in, int fmt, const char *optstr)
{
	jas_image_fmtinfo_t *fmtinfo;
	jas_image_t *image;

	image = 0;

	/* If possible, try to determine the format of the input data. */
	if (fmt < 0) {
		if ((fmt = jas_image_getfmt(in)) < 0)
			goto error;
	}

	/* Is it possible to decode an image represented in this format? */
	if (!(fmtinfo = jas_image_lookupfmtbyid(fmt)))
		goto error;
	if (!fmtinfo->ops.decode)
		goto error;

	/* Decode the image. */
	if (!(image = (*fmtinfo->ops.decode)(in, optstr)))
		goto error;

	/* Create a color profile if needed. */
	if (!jas_clrspc_isunknown(image->clrspc_) &&
	  !jas_clrspc_isgeneric(image->clrspc_) && !image->cmprof_) {
		if (!(image->cmprof_ =
		  jas_cmprof_createfromclrspc(jas_image_clrspc(image))))
			goto error;
	}

	return image;
error:
	if (image)
		jas_image_destroy(image);
	return 0;
}