CVE-2016-10248: NULL Pointer Dereference (CWE-476)
value: The jpc_tsfb_synthesize function in jpc_tsfb.c in JasPer before 1.900.9 allows remote attackers to cause a denial of service (NULL pointer dereference) via vectors involving an empty sequence.
CWE description: The product dereferences a pointer that it expects to be valid but is NULL.
CVSS v3 base score: 7.5
Published: 2017-03-15T14:59Z
Repository: https://github.com/mdadams/jasper
Language: C
Files changed: src/libjasper/include/jasper/jas_math.h, src/libjasper/include/jasper/jas_seq.h, src/libjasper/jpc/jpc_dec.c, src/libjasper/jpc/jpc_tsfb.c
Commit date: 2016-10-19 17:57:40 -0700
Commit message: Fixed an integral type promotion problem by adding a JAS_CAST.
Modified the jpc_tsfb_synthesize function so that it will be a noop for
an empty sequence (in order to avoid dereferencing a null pointer).

Diff: diff --git a/src/libjasper/include/jasper/jas_math.h b/src/libjasper/include/jasper/jas_math.h
index 3f1a933..91efc60 100644
--- a/src/libjasper/include/jasper/jas_math.h
+++ b/src/libjasper/include/jasper/jas_math.h
@@ -110,10 +110,28 @@ extern "C" {
 /* Compute the integer with the specified number of least significant bits
   set to one. */
 #define	JAS_ONES(n) \
   ((1 << (n)) - 1)
 
+/******************************************************************************\
+*
+\******************************************************************************/
+
+__attribute__((no_sanitize("undefined")))
+inline static jas_int_asr(int x, int n)
+{
+	assert(n >= 0);
+	return x >> n;
+}
+
+__attribute__((no_sanitize("undefined")))
+inline static jas_int_asl(int x, int n)
+{
+	assert(n >= 0);
+	return x << n;
+}
+
 /******************************************************************************\
 * Safe integer arithmetic (i.e., with overflow checking).
 \******************************************************************************/
 
 /* Compute the product of two size_t integers with overflow checking. */
diff --git a/src/libjasper/include/jasper/jas_seq.h b/src/libjasper/include/jasper/jas_seq.h
index e96548e..f470c84 100644
--- a/src/libjasper/include/jasper/jas_seq.h
+++ b/src/libjasper/include/jasper/jas_seq.h
@@ -152,10 +152,13 @@ typedef jas_matrix_t jas_seq_t;
 
 /* Get the number of columns. */
 #define jas_matrix_numcols(matrix) \
 	((matrix)->numcols_)
 
+#define jas_matrix_size(matrix) \
+	(jas_matrix_width(matrix) * jas_matrix_height(matrix))
+
 /* Get a matrix element. */
 #define jas_matrix_get(matrix, i, j) \
 	((matrix)->rows_[i][j])
 
 /* Set a matrix element. */
@@ -267,10 +270,12 @@ jas_matrix_t *jas_seq2d_create(int xstart, int ystart, int xend, int yend);
 	((s)->yend_ - (s)->ystart_)
 #define	jas_seq2d_setshift(s, x, y) \
 	((s)->xstart_ = (x), (s)->ystart_ = (y), \
 	  (s)->xend_ = (s)->xstart_ + (s)->numcols_, \
 	  (s)->yend_ = (s)->ystart_ + (s)->numrows_)
+#define jas_seq2d_size(s) \
+	(jas_seq2d_width(s) * jas_seq2d_height(s))
 
 void jas_seq2d_bindsub(jas_matrix_t *s, jas_matrix_t *s1, int xstart,
   int ystart, int xend, int yend);
 
 /******************************************************************************\
diff --git a/src/libjasper/jpc/jpc_dec.c b/src/libjasper/jpc/jpc_dec.c
index 0d38c62..755c1c6 100644
--- a/src/libjasper/jpc/jpc_dec.c
+++ b/src/libjasper/jpc/jpc_dec.c
@@ -1836,10 +1836,17 @@ static void jpc_undo_roi(jas_matrix_t *x, int roishift, int bgshift, int numbps)
 	jpc_fix_t val;
 	jpc_fix_t mag;
 	bool warn;
 	uint_fast32_t mask;
 
+	if (roishift < 0) {
+		/* We could instead return an error here. */
+		/* I do not think it matters much. */
+		jas_eprintf("warning: forcing negative ROI shift to zero "
+		  "(bitstream is probably corrupt)\n");
+		roishift = 0;
+	}
 	if (roishift == 0 && bgshift == 0) {
 		return;
 	}
 	thresh = 1 << roishift;
 
@@ -1854,11 +1861,11 @@ static void jpc_undo_roi(jas_matrix_t *x, int roishift, int bgshift, int numbps)
 				val = (val < 0) ? (-mag) : mag;
 				jas_matrix_set(x, i, j, val);
 			} else {
 				/* We are dealing with non-ROI (i.e., background) data. */
 				mag <<= bgshift;
-				mask = (1 << numbps) - 1;
+				mask = (JAS_CAST(uint_fast32_t, 1) << numbps) - 1;
 				/* Perform a basic sanity check on the sample value. */
 				/* Some implementations write garbage in the unused
 				  most-significant bit planes introduced by ROI shifting.
 				  Here we ensure that any such bits are masked off. */
 				if (mag & (~mask)) {
diff --git a/src/libjasper/jpc/jpc_tsfb.c b/src/libjasper/jpc/jpc_tsfb.c
index 7aa293f..b51b747 100644
--- a/src/libjasper/jpc/jpc_tsfb.c
+++ b/src/libjasper/jpc/jpc_tsfb.c
@@ -146,11 +146,12 @@ int jpc_tsfb_analyze2(jpc_tsfb_t *tsfb, int *a, int xstart, int ystart,
 	return 0;
 }
 
 int jpc_tsfb_synthesize(jpc_tsfb_t *tsfb, jas_seq2d_t *a)
 {
-	return (tsfb->numlvls > 0) ? jpc_tsfb_synthesize2(tsfb,
+	return (tsfb->numlvls > 0 && jas_seq2d_size(a)) ?
+	  jpc_tsfb_synthesize2(tsfb,
 	  jas_seq2d_getref(a, jas_seq2d_xstart(a), jas_seq2d_ystart(a)),
 	  jas_seq2d_xstart(a), jas_seq2d_ystart(a), jas_seq2d_width(a),
 	  jas_seq2d_height(a), jas_seq2d_rowstep(a), tsfb->numlvls - 1) : 0;
 }
 

Vulnerable code: 				mask = (1 << numbps) - 1;
	return (tsfb->numlvls > 0) ? jpc_tsfb_synthesize2(tsfb,
Fixed code: /******************************************************************************\
*
\******************************************************************************/

__attribute__((no_sanitize("undefined")))
inline static jas_int_asr(int x, int n)
{
	assert(n >= 0);
	return x >> n;
}

__attribute__((no_sanitize("undefined")))
inline static jas_int_asl(int x, int n)
{
	assert(n >= 0);
	return x << n;
}

#define jas_matrix_size(matrix) \
	(jas_matrix_width(matrix) * jas_matrix_height(matrix))

#define jas_seq2d_size(s) \
	(jas_seq2d_width(s) * jas_seq2d_height(s))
	if (roishift < 0) {
		/* We could instead return an error here. */
		/* I do not think it matters much. */
		jas_eprintf("warning: forcing negative ROI shift to zero "
		  "(bitstream is probably corrupt)\n");
		roishift = 0;
	}
				mask = (JAS_CAST(uint_fast32_t, 1) << numbps) - 1;
	return (tsfb->numlvls > 0 && jas_seq2d_size(a)) ?
	  jpc_tsfb_synthesize2(tsfb,