CVE-2016-8654: Improper Restriction of Operations within the Bounds of a Memory Buffer (CWE-119)
value: A heap-buffer overflow vulnerability was found in QMFB code in JPC codec caused by buffer being allocated with too small size. jasper versions before 2.0.0 are affected.
CWE description: The product performs operations on a memory buffer, but it reads from or writes to a memory location outside the buffer's intended boundary. This may result in read or write operations on unexpected memory locations that could be linked to other variables, data structures, or internal program data.
CVSS v3 base score: 7.8
Published: 2018-08-01T16:29Z
Repository: https://github.com/mdadams/jasper
Language: C
Files changed: src/libjasper/jpc/jpc_qmfb.c
Commit date: 2016-11-26 20:23:23 -0800
Commit message: Fixed a buffer overrun problem in the QMFB code in the JPC codec
that was caused by a buffer being allocated with a size that was too small
in some cases.
Added a new regression test case.

Diff: diff --git a/data/test/bad/PoC1.jpc b/data/test/bad/PoC1.jpc
new file mode 100644
index 0000000..9523954
Binary files /dev/null and b/data/test/bad/PoC1.jpc differ
diff --git a/src/libjasper/jpc/jpc_qmfb.c b/src/libjasper/jpc/jpc_qmfb.c
index 45be24e..f26070f 100644
--- a/src/libjasper/jpc/jpc_qmfb.c
+++ b/src/libjasper/jpc/jpc_qmfb.c
@@ -372,24 +372,24 @@ void jpc_qmfb_split_col(jpc_fix_t *a, int numrows, int stride,
 	jpc_fix_t *buf = splitbuf;
 	register jpc_fix_t *srcptr;
 	register jpc_fix_t *dstptr;
 	register int n;
 	register int m;
-	int hstartcol;
+	int hstartrow;
 
 	/* Get a buffer. */
 	if (bufsize > QMFB_SPLITBUFSIZE) {
 		if (!(buf = jas_alloc2(bufsize, sizeof(jpc_fix_t)))) {
 			/* We have no choice but to commit suicide in this case. */
 			abort();
 		}
 	}
 
 	if (numrows >= 2) {
-		hstartcol = (numrows + 1 - parity) >> 1;
-		// ORIGINAL (WRONG): m = (parity) ? hstartcol : (numrows - hstartcol);
-		m = numrows - hstartcol;
+		hstartrow = (numrows + 1 - parity) >> 1;
+		// ORIGINAL (WRONG): m = (parity) ? hstartrow : (numrows - hstartrow);
+		m = numrows - hstartrow;
 
 		/* Save the samples destined for the highpass channel. */
 		n = m;
 		dstptr = buf;
 		srcptr = &a[(1 - parity) * stride];
@@ -406,11 +406,11 @@ void jpc_qmfb_split_col(jpc_fix_t *a, int numrows, int stride,
 			*dstptr = *srcptr;
 			dstptr += stride;
 			srcptr += stride << 1;
 		}
 		/* Copy the saved samples into the highpass channel. */
-		dstptr = &a[hstartcol * stride];
+		dstptr = &a[hstartrow * stride];
 		srcptr = buf;
 		n = m;
 		while (n-- > 0) {
 			*dstptr = *srcptr;
 			dstptr += stride;
@@ -437,24 +437,25 @@ void jpc_qmfb_split_colgrp(jpc_fix_t *a, int numrows, int stride,
 	register jpc_fix_t *srcptr2;
 	register jpc_fix_t *dstptr2;
 	register int n;
 	register int i;
 	int m;
-	int hstartcol;
+	int hstartrow;
 
 	/* Get a buffer. */
 	if (bufsize > QMFB_SPLITBUFSIZE) {
-		if (!(buf = jas_alloc2(bufsize, sizeof(jpc_fix_t)))) {
+		if (!(buf = jas_alloc3(bufsize, JPC_QMFB_COLGRPSIZE,
+		  sizeof(jpc_fix_t)))) {
 			/* We have no choice but to commit suicide in this case. */
 			abort();
 		}
 	}
 
 	if (numrows >= 2) {
-		hstartcol = (numrows + 1 - parity) >> 1;
-		// ORIGINAL (WRONG): m = (parity) ? hstartcol : (numrows - hstartcol);
-		m = numrows - hstartcol;
+		hstartrow = (numrows + 1 - parity) >> 1;
+		// ORIGINAL (WRONG): m = (parity) ? hstartrow : (numrows - hstartrow);
+		m = numrows - hstartrow;
 
 		/* Save the samples destined for the highpass channel. */
 		n = m;
 		dstptr = buf;
 		srcptr = &a[(1 - parity) * stride];
@@ -483,11 +484,11 @@ void jpc_qmfb_split_colgrp(jpc_fix_t *a, int numrows, int stride,
 			}
 			dstptr += stride;
 			srcptr += stride << 1;
 		}
 		/* Copy the saved samples into the highpass channel. */
-		dstptr = &a[hstartcol * stride];
+		dstptr = &a[hstartrow * stride];
 		srcptr = buf;
 		n = m;
 		while (n-- > 0) {
 			dstptr2 = dstptr;
 			srcptr2 = srcptr;
@@ -524,11 +525,11 @@ void jpc_qmfb_split_colres(jpc_fix_t *a, int numrows, int numcols,
 	int m;
 	int hstartcol;
 
 	/* Get a buffer. */
 	if (bufsize > QMFB_SPLITBUFSIZE) {
-		if (!(buf = jas_alloc2(bufsize, sizeof(jpc_fix_t)))) {
+		if (!(buf = jas_alloc3(bufsize, numcols, sizeof(jpc_fix_t)))) {
 			/* We have no choice but to commit suicide in this case. */
 			abort();
 		}
 	}
 
@@ -719,11 +720,12 @@ void jpc_qmfb_join_colgrp(jpc_fix_t *a, int numrows, int stride,
 	register int i;
 	int hstartcol;
 
 	/* Allocate memory for the join buffer from the heap. */
 	if (bufsize > QMFB_JOINBUFSIZE) {
-		if (!(buf = jas_alloc3(bufsize, JPC_QMFB_COLGRPSIZE, sizeof(jpc_fix_t)))) {
+		if (!(buf = jas_alloc3(bufsize, JPC_QMFB_COLGRPSIZE,
+		  sizeof(jpc_fix_t)))) {
 			/* We have no choice but to commit suicide. */
 			abort();
 		}
 	}
 

Vulnerable code: 	int hstartcol;
		hstartcol = (numrows + 1 - parity) >> 1;
		// ORIGINAL (WRONG): m = (parity) ? hstartcol : (numrows - hstartcol);
		m = numrows - hstartcol;
		dstptr = &a[hstartcol * stride];
	int hstartcol;
		if (!(buf = jas_alloc2(bufsize, sizeof(jpc_fix_t)))) {
		hstartcol = (numrows + 1 - parity) >> 1;
		// ORIGINAL (WRONG): m = (parity) ? hstartcol : (numrows - hstartcol);
		m = numrows - hstartcol;
		dstptr = &a[hstartcol * stride];
		if (!(buf = jas_alloc2(bufsize, sizeof(jpc_fix_t)))) {
		if (!(buf = jas_alloc3(bufsize, JPC_QMFB_COLGRPSIZE, sizeof(jpc_fix_t)))) {
Fixed code: 	int hstartrow;
		hstartrow = (numrows + 1 - parity) >> 1;
		// ORIGINAL (WRONG): m = (parity) ? hstartrow : (numrows - hstartrow);
		m = numrows - hstartrow;
		dstptr = &a[hstartrow * stride];
	int hstartrow;
		if (!(buf = jas_alloc3(bufsize, JPC_QMFB_COLGRPSIZE,
		  sizeof(jpc_fix_t)))) {
		hstartrow = (numrows + 1 - parity) >> 1;
		// ORIGINAL (WRONG): m = (parity) ? hstartrow : (numrows - hstartrow);
		m = numrows - hstartrow;
		dstptr = &a[hstartrow * stride];
		if (!(buf = jas_alloc3(bufsize, numcols, sizeof(jpc_fix_t)))) {
		if (!(buf = jas_alloc3(bufsize, JPC_QMFB_COLGRPSIZE,
		  sizeof(jpc_fix_t)))) {