Commit message: ras_enc: check components for RGB, fixes NULL pointer dereference

Fixes CVE-2018-18873

Closes https://github.com/jasper-maint/jasper/issues/15
Closes https://github.com/mdadams/jasper/issues/184
CVE: CVE-2018-18873
CWE: CWE-476
Vulnerability description: An issue was discovered in JasPer 2.0.14. There is a NULL pointer dereference in the function ras_putdatastd in ras/ras_enc.c.

Project: jasper
File: ras_enc.c

static int ras_putint(jas_stream_t *out, int_fast32_t val)
{
	int_fast32_t x;
	int i;
	int c;

	assert(val >= 0);
	x = val;
	for (i = 0; i < 4; i++) {
		c = (x >> (8 * (3 - i))) & 0xff;
		if (jas_stream_putc(out, c) == EOF) {
			return -1;
		}
	}

	return 0;
}

Project: jasper
File: ras_enc.c

int ras_encode(jas_image_t *image, jas_stream_t *out, const char *optstr)
{
	int_fast32_t width;
	int_fast32_t height;
	int_fast32_t depth;
	int cmptno;
	int i;
	ras_hdr_t hdr;
	int rowsize;
	ras_enc_t encbuf;
	ras_enc_t *enc = &encbuf;

	JAS_DBGLOG(10, ("ras_encode(%p, %p, \"%s\"\n", image, out,
	  optstr ? optstr : ""));

	if (optstr) {
		jas_eprintf("warning: ignoring RAS encoder options\n");
	}

	switch (jas_clrspc_fam(jas_image_clrspc(image))) {
	case JAS_CLRSPC_FAM_RGB:
		if (jas_image_clrspc(image) != JAS_CLRSPC_SRGB)
			jas_eprintf("warning: inaccurate color\n");
		enc->numcmpts = 3;
		if ((enc->cmpts[0] = jas_image_getcmptbytype(image,
		  JAS_IMAGE_CT_COLOR(JAS_CLRSPC_CHANIND_RGB_R))) < 0 ||
		  (enc->cmpts[1] = jas_image_getcmptbytype(image,
		  JAS_IMAGE_CT_COLOR(JAS_CLRSPC_CHANIND_RGB_G))) < 0 ||
		  (enc->cmpts[2] = jas_image_getcmptbytype(image,
		  JAS_IMAGE_CT_COLOR(JAS_CLRSPC_CHANIND_RGB_B))) < 0) {
			jas_eprintf("error: missing color component\n");
			return -1;
		}
		break;
	case JAS_CLRSPC_FAM_GRAY:
		if (jas_image_clrspc(image) != JAS_CLRSPC_SGRAY)
			jas_eprintf("warning: inaccurate color\n");
		enc->numcmpts = 1;
		if ((enc->cmpts[0] = jas_image_getcmptbytype(image,
		  JAS_IMAGE_CT_COLOR(JAS_CLRSPC_CHANIND_GRAY_Y))) < 0) {
			jas_eprintf("error: missing color component\n");
			return -1;
		}
		break;
	default:
		jas_eprintf("error: unsupported color space\n");
		return -1;
		break;
	}

	width = jas_image_cmptwidth(image, enc->cmpts[0]);
	height = jas_image_cmptheight(image, enc->cmpts[0]);
	depth = jas_image_cmptprec(image, enc->cmpts[0]);

	for (cmptno = 0; cmptno < enc->numcmpts; ++cmptno) {
		if (jas_image_cmptwidth(image, enc->cmpts[cmptno]) != width ||
		  jas_image_cmptheight(image, enc->cmpts[cmptno]) != height ||
		  jas_image_cmptprec(image, enc->cmpts[cmptno]) != depth ||
		  jas_image_cmptsgnd(image, enc->cmpts[cmptno]) != false ||
		  jas_image_cmpttlx(image, enc->cmpts[cmptno]) != 0 ||
		  jas_image_cmpttly(image, enc->cmpts[cmptno]) != 0) {
			jas_eprintf("The RAS format cannot be used to represent an image with this geometry.\n");
			return -1;
		}
	}

	/* Ensure that the image can be encoded in the desired format. */
	if (enc->numcmpts == 3) {

		/* All three components must have the same subsampling
		  factor and have a precision of eight bits. */
		if (enc->numcmpts > 1) {
			for (i = 0; i < enc->numcmpts; ++i) {
				if (jas_image_cmptprec(image, enc->cmpts[i]) != 8) {
					return -1;
				}
			}
		}
	} else if (enc->numcmpts == 1) {
		/* NOP */
	} else {
		return -1;
	}

	hdr.magic = RAS_MAGIC;
	hdr.width = width;
	hdr.height = height;
	hdr.depth = (enc->numcmpts == 3) ? 24 : depth;

	rowsize = RAS_ROWSIZE(&hdr);
	hdr.length = rowsize * hdr.height;
	hdr.type = RAS_TYPE_STD;

	hdr.maptype = RAS_MT_NONE;
	hdr.maplength = 0;

	if (ras_puthdr(out, &hdr)) {
		return -1;
	}

	if (ras_putdata(out, &hdr, image, enc->numcmpts, enc->cmpts)) {
		return -1;
	}

	return 0;
}

Project: jasper
File: ras_enc.c

static int ras_putdatastd(jas_stream_t *out, ras_hdr_t *hdr, jas_image_t *image,  int numcmpts, int *cmpts)
{
	int rowsize;
	int pad;
	unsigned int z;
	int nz;
	int c;
	int x;
	int y;
	int v;
	jas_matrix_t *data[3];
	int i;

	assert(numcmpts <= 3);

	if (RAS_ISRGB(hdr) && numcmpts < 3) {
		/* need 3 components for RGB */
		return -1;
	}

	for (i = 0; i < 3; ++i) {
		data[i] = 0;
	}

	for (i = 0; i < numcmpts; ++i) {
		if (!(data[i] = jas_matrix_create(jas_image_height(image),
		  jas_image_width(image)))) {
			goto error;
		}
	}

	rowsize = RAS_ROWSIZE(hdr);
	pad = rowsize - (hdr->width * hdr->depth + 7) / 8;

	hdr->length = hdr->height * rowsize;

	for (y = 0; y < hdr->height; y++) {
		for (i = 0; i < numcmpts; ++i) {
			if (jas_image_readcmpt(image, cmpts[i], 0, y,
					jas_image_width(image), 1, data[i])) {
				goto error;
			}
		}
		z = 0;
		nz = 0;
		for (x = 0; x < hdr->width; x++) {
			z <<= hdr->depth;
			if (RAS_ISRGB(hdr)) {
				v = RAS_RED((jas_matrix_getv(data[0], x))) |
				  RAS_GREEN((jas_matrix_getv(data[1], x))) |
				  RAS_BLUE((jas_matrix_getv(data[2], x)));
			} else {
				v = (jas_matrix_getv(data[0], x));
			}
			z |= v & RAS_ONES(hdr->depth);
			nz += hdr->depth;
			while (nz >= 8) {
				c = (z >> (nz - 8)) & 0xff;
				if (jas_stream_putc(out, c) == EOF) {
					goto error;
				}
				nz -= 8;
				z &= RAS_ONES(nz);
			}
		}
		if (nz > 0) {
			c = (z >> (8 - nz)) & RAS_ONES(nz);
			if (jas_stream_putc(out, c) == EOF) {
				goto error;
			}
		}
		if (pad % 2) {
			if (jas_stream_putc(out, 0) == EOF) {
				goto error;
			}
		}
	}

	for (i = 0; i < numcmpts; ++i) {
		jas_matrix_destroy(data[i]);
		data[i] = 0;
	}

	return 0;

error:
	for (i = 0; i < numcmpts; ++i) {
		if (data[i]) {
			jas_matrix_destroy(data[i]);
		}
	}
	return -1;
}