CVE-2017-6308: Integer Overflow or Wraparound (CWE-190)
value: An issue was discovered in tnef before 1.4.13. Several Integer Overflows, which can lead to Heap Overflows, have been identified in the functions that wrap memory allocation.
CWE description: The product performs a calculation that can
         produce an integer overflow or wraparound when the logic
         assumes that the resulting value will always be larger than
         the original value. This occurs when an integer value is
         incremented to a value that is too large to store in the
         associated representation. When this occurs, the value may
         become a very small or negative number.
CVSS v3 base score: 7.8
Published: 2017-02-24T04:59Z
Repository: https://github.com/verdammelt/tnef
Language: C
Files changed: src/alloc.c, src/alloc.h
Commit date: 2017-02-22 20:51:35 -0500
Commit message: Fix integer overflows and harden memory allocator.

Diff: diff --git a/src/alloc.c b/src/alloc.c
index 47d192f..c23909c 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -38,70 +38,93 @@ size_t
 get_alloc_limit()
 {
     return alloc_limit;
 }
 
+size_t
+check_mul_overflow(size_t a, size_t b, size_t* res)
+{
+    size_t tmp = a * b;
+    if (a != 0 && tmp / a != b) return 1;
+    *res = tmp;
+    return 0;
+}
+
 static void
 alloc_limit_failure (char *fn_name, size_t size)
 {
-    fprintf (stderr, 
+    fprintf (stderr,
              "%s: Maximum allocation size exceeded "
              "(maxsize = %lu; size = %lu).\n",
              fn_name,
-             (unsigned long)alloc_limit, 
+             (unsigned long)alloc_limit,
              (unsigned long)size);
 }
 
 void
 alloc_limit_assert (char *fn_name, size_t size)
 {
     if (alloc_limit && size > alloc_limit)
     {
-	alloc_limit_failure (fn_name, size);
-	exit (-1);
+        alloc_limit_failure (fn_name, size);
+        exit (-1);
     }
 }
 
 /* attempts to malloc memory, if fails print error and call abort */
 void*
-xmalloc (size_t size)
+xmalloc (size_t num, size_t size)
 {
-    void *ptr = malloc (size);
-    if (!ptr 
+    size_t res;
+    if (check_mul_overflow(num, size, &res))
+        abort();
+
+    void *ptr = malloc (res);
+    if (!ptr
         && (size != 0))         /* some libc don't like size == 0 */
     {
         perror ("xmalloc: Memory allocation failure");
         abort();
     }
     return ptr;
 }
 
 /* Allocates memory but only up to a limit */
 void*
-checked_xmalloc (size_t size)
+checked_xmalloc (size_t num, size_t size)
 {
-    alloc_limit_assert ("checked_xmalloc", size);
-    return xmalloc (size);
+    size_t res;
+    if (check_mul_overflow(num, size, &res))
+        abort();
+
+    alloc_limit_assert ("checked_xmalloc", res);
+    return xmalloc (num, size);
 }
 
 /* xmallocs memory and clears it out */
 void*
 xcalloc (size_t num, size_t size)
 {
-    void *ptr = malloc(num * size);
+    size_t res;
+    if (check_mul_overflow(num, size, &res))
+        abort();
+
+    void *ptr;
+    ptr = malloc(res);
     if (ptr)
     {
-        memset (ptr, '\0', (num * size));
+        memset (ptr, '\0', (res));
     }
     return ptr;
 }
 
 /* xcallocs memory but only up to a limit */
 void*
 checked_xcalloc (size_t num, size_t size)
 {
-    alloc_limit_assert ("checked_xcalloc", (num *size));
+    size_t res;
+    if (check_mul_overflow(num, size, &res))
+        abort();
+
+    alloc_limit_assert ("checked_xcalloc", (res));
     return xcalloc (num, size);
 }
-
-
-
diff --git a/src/alloc.h b/src/alloc.h
index d63f6a3..8fbd6bf 100644
--- a/src/alloc.h
+++ b/src/alloc.h
@@ -33,22 +33,22 @@ extern void free (void*);
 #endif /* STDC_HEADERS */
 
 extern void set_alloc_limit (size_t size);
 extern size_t get_alloc_limit();
 extern void alloc_limit_assert (char *fn_name, size_t size);
-extern void* checked_xmalloc (size_t size);
-extern void* xmalloc (size_t size);
+extern void* checked_xmalloc (size_t num, size_t size);
+extern void* xmalloc (size_t num, size_t size);
 extern void* checked_xcalloc (size_t num, size_t size);
 extern void* xcalloc (size_t num, size_t size);
 
 #define XMALLOC(_type,_num)			                \
-        ((_type*)xmalloc((_num)*sizeof(_type)))
+        ((_type*)xmalloc((_num), sizeof(_type)))
 #define XCALLOC(_type,_num) 				        \
         ((_type*)xcalloc((_num), sizeof (_type)))
 #define CHECKED_XMALLOC(_type,_num) 			        \
-        ((_type*)checked_xmalloc((_num)*sizeof(_type)))
+        ((_type*)checked_xmalloc((_num),sizeof(_type)))
 #define CHECKED_XCALLOC(_type,_num) 			        \
         ((_type*)checked_xcalloc((_num),sizeof(_type)))
 #define XFREE(_ptr)						\
-	do { if (_ptr) { free (_ptr); _ptr = 0; } } while (0)
+        do { if (_ptr) { free (_ptr); _ptr = 0; } } while (0)
 
 #endif /* ALLOC_H */

Vulnerable code:     fprintf (stderr, 
             (unsigned long)alloc_limit, 
	alloc_limit_failure (fn_name, size);
	exit (-1);
xmalloc (size_t size)
    void *ptr = malloc (size);
    if (!ptr 
checked_xmalloc (size_t size)
    alloc_limit_assert ("checked_xmalloc", size);
    return xmalloc (size);
    void *ptr = malloc(num * size);
        memset (ptr, '\0', (num * size));
    alloc_limit_assert ("checked_xcalloc", (num *size));



extern void* checked_xmalloc (size_t size);
extern void* xmalloc (size_t size);
        ((_type*)xmalloc((_num)*sizeof(_type)))
        ((_type*)checked_xmalloc((_num)*sizeof(_type)))
	do { if (_ptr) { free (_ptr); _ptr = 0; } } while (0)
Fixed code: size_t
check_mul_overflow(size_t a, size_t b, size_t* res)
{
    size_t tmp = a * b;
    if (a != 0 && tmp / a != b) return 1;
    *res = tmp;
    return 0;
}

    fprintf (stderr,
             (unsigned long)alloc_limit,
        alloc_limit_failure (fn_name, size);
        exit (-1);
xmalloc (size_t num, size_t size)
    size_t res;
    if (check_mul_overflow(num, size, &res))
        abort();

    void *ptr = malloc (res);
    if (!ptr
checked_xmalloc (size_t num, size_t size)
    size_t res;
    if (check_mul_overflow(num, size, &res))
        abort();

    alloc_limit_assert ("checked_xmalloc", res);
    return xmalloc (num, size);
    size_t res;
    if (check_mul_overflow(num, size, &res))
        abort();

    void *ptr;
    ptr = malloc(res);
        memset (ptr, '\0', (res));
    size_t res;
    if (check_mul_overflow(num, size, &res))
        abort();

    alloc_limit_assert ("checked_xcalloc", (res));
extern void* checked_xmalloc (size_t num, size_t size);
extern void* xmalloc (size_t num, size_t size);
        ((_type*)xmalloc((_num), sizeof(_type)))
        ((_type*)checked_xmalloc((_num),sizeof(_type)))
        do { if (_ptr) { free (_ptr); _ptr = 0; } } while (0)