Vulnerability: Stack-based buffer overflow in the color_string_to_rgba function in libavcodec/xpmdec.c in FFmpeg 3.3 before 3.3.1 allows remote attackers to cause a denial of service (application crash) or possibly have unspecified other impact via a crafted file.
Vulnerability ID: CVE-2017-9990
CWE: CWE-119
Severity score: 6.4
Project: https://github.com/FFmpeg/FFmpeg
Published: 2017-06-28
File: libavcodec/xpmdec.c
Commit message: avcodec/xpmdec: Fix multiple pointer/memory issues

Most of these were found through code review in response to
fixing 1466/clusterfuzz-testcase-minimized-5961584419536896
There is thus no testcase for most of this.
The initial issue was Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Commit author: Michael Niedermayer

Vulnerable version (before fix):
 typedef struct XPMContext {     uint32_t  *pixels;     int        pixels_size; } XPMDecContext;  typedef struct ColorEntry { static uint32_t color_string_to_rgba(const char *p, int len)     const ColorEntry *entry;     char color_name[100];      if (*p == '#') {         p++;         len--; static int xpm_decode_frame(AVCodecContext *avctx, void *data, {     XPMDecContext *x = avctx->priv_data;     AVFrame *p=data;     const uint8_t *end, *ptr = avpkt->data;     int ncolors, cpp, ret, i, j;     int64_t size;     uint32_t *dst;      avctx->pix_fmt = AV_PIX_FMT_BGRA;      end = avpkt->data + avpkt->size;     while (memcmp(ptr, "/* XPM */", 9) && ptr < end - 9)         ptr++;      if (ptr >= end) {         av_log(avctx, AV_LOG_ERROR, "missing signature\n");         return AVERROR_INVALIDDATA;     } static int xpm_decode_frame(AVCodecContext *avctx, void *data,      size = 1;     for (i = 0; i < cpp; i++)         size *= 94;      if (ncolors <= 0 || ncolors > size) {         av_log(avctx, AV_LOG_ERROR, "invalid number of colors: %d\n", ncolors); static int xpm_decode_frame(AVCodecContext *avctx, void *data,         return AVERROR(ENOMEM);      ptr += mod_strcspn(ptr, ",") + 1;     for (i = 0; i < ncolors; i++) {         const uint8_t *index;         int len;          ptr += mod_strcspn(ptr, "\"") + 1;         if (ptr + cpp > end)             return AVERROR_INVALIDDATA;         index = ptr;         ptr += cpp; static int xpm_decode_frame(AVCodecContext *avctx, void *data,          x->pixels[ret] = color_string_to_rgba(ptr, len);         ptr += mod_strcspn(ptr, ",") + 1;     }      for (i = 0; i < avctx->height; i++) {         dst = (uint32_t *)(p->data[0] + i * p->linesize[0]);         ptr += mod_strcspn(ptr, "\"") + 1;          for (j = 0; j < avctx->width; j++) {             if (ptr + cpp > end)                 return AVERROR_INVALIDDATA;              if ((ret = ascii2index(ptr, cpp)) < 0) static av_cold int xpm_decode_close(AVCodecContext *avctx)     XPMDecContext *x = avctx->priv_data;     av_freep(&x->pixels);      return 0; } 

Fixed version (after fix):
 typedef struct XPMContext {     uint32_t  *pixels;     int        pixels_size;     uint8_t   *buf;     int        buf_size; } XPMDecContext;  typedef struct ColorEntry { static uint32_t color_string_to_rgba(const char *p, int len)     const ColorEntry *entry;     char color_name[100];      len = FFMIN(FFMAX(len, 0), sizeof(color_name) - 1);      if (*p == '#') {         p++;         len--; static int xpm_decode_frame(AVCodecContext *avctx, void *data, {     XPMDecContext *x = avctx->priv_data;     AVFrame *p=data;     const uint8_t *end, *ptr;     int ncolors, cpp, ret, i, j;     int64_t size;     uint32_t *dst;      avctx->pix_fmt = AV_PIX_FMT_BGRA;      av_fast_padded_malloc(&x->buf, &x->buf_size, avpkt->size);     if (!x->buf)         return AVERROR(ENOMEM);     memcpy(x->buf, avpkt->data, avpkt->size);     x->buf[avpkt->size] = 0;      ptr = x->buf;     end = x->buf + avpkt->size;     while (end - ptr > 9 && memcmp(ptr, "/* XPM */", 9))         ptr++;      if (end - ptr <= 9) {         av_log(avctx, AV_LOG_ERROR, "missing signature\n");         return AVERROR_INVALIDDATA;     } static int xpm_decode_frame(AVCodecContext *avctx, void *data,      size = 1;     for (i = 0; i < cpp; i++)         size *= 95;      if (ncolors <= 0 || ncolors > size) {         av_log(avctx, AV_LOG_ERROR, "invalid number of colors: %d\n", ncolors); static int xpm_decode_frame(AVCodecContext *avctx, void *data,         return AVERROR(ENOMEM);      ptr += mod_strcspn(ptr, ",") + 1;     if (end - ptr < 1)         return AVERROR_INVALIDDATA;      for (i = 0; i < ncolors; i++) {         const uint8_t *index;         int len;          ptr += mod_strcspn(ptr, "\"") + 1;         if (end - ptr < cpp)             return AVERROR_INVALIDDATA;         index = ptr;         ptr += cpp; static int xpm_decode_frame(AVCodecContext *avctx, void *data,          x->pixels[ret] = color_string_to_rgba(ptr, len);         ptr += mod_strcspn(ptr, ",") + 1;         if (end - ptr < 1)             return AVERROR_INVALIDDATA;     }      for (i = 0; i < avctx->height; i++) {         dst = (uint32_t *)(p->data[0] + i * p->linesize[0]);         if (end - ptr < 1)             return AVERROR_INVALIDDATA;         ptr += mod_strcspn(ptr, "\"") + 1;         if (end - ptr < 1)             return AVERROR_INVALIDDATA;          for (j = 0; j < avctx->width; j++) {             if (end - ptr < cpp)                 return AVERROR_INVALIDDATA;              if ((ret = ascii2index(ptr, cpp)) < 0) static av_cold int xpm_decode_close(AVCodecContext *avctx)     XPMDecContext *x = avctx->priv_data;     av_freep(&x->pixels);      av_freep(&x->buf);     x->buf_size = 0;      return 0; } 

Patch:
patch: @@ -29,6 +29,8 @@
 typedef struct XPMContext {
     uint32_t  *pixels;
     int        pixels_size;
+    uint8_t   *buf;
+    int        buf_size;
 } XPMDecContext;
 
 typedef struct ColorEntry {
@@ -233,6 +235,8 @@ static uint32_t color_string_to_rgba(const char *p, int len)
     const ColorEntry *entry;
     char color_name[100];
 
+    len = FFMIN(FFMAX(len, 0), sizeof(color_name) - 1);
+
     if (*p == '#') {
         p++;
         len--;
@@ -299,18 +303,25 @@ static int xpm_decode_frame(AVCodecContext *avctx, void *data,
 {
     XPMDecContext *x = avctx->priv_data;
     AVFrame *p=data;
-    const uint8_t *end, *ptr = avpkt->data;
+    const uint8_t *end, *ptr;
     int ncolors, cpp, ret, i, j;
     int64_t size;
     uint32_t *dst;
 
     avctx->pix_fmt = AV_PIX_FMT_BGRA;
 
-    end = avpkt->data + avpkt->size;
-    while (memcmp(ptr, "/* XPM */", 9) && ptr < end - 9)
+    av_fast_padded_malloc(&x->buf, &x->buf_size, avpkt->size);
+    if (!x->buf)
+        return AVERROR(ENOMEM);
+    memcpy(x->buf, avpkt->data, avpkt->size);
+    x->buf[avpkt->size] = 0;
+
+    ptr = x->buf;
+    end = x->buf + avpkt->size;
+    while (end - ptr > 9 && memcmp(ptr, "/* XPM */", 9))
         ptr++;
 
-    if (ptr >= end) {
+    if (end - ptr <= 9) {
         av_log(avctx, AV_LOG_ERROR, "missing signature\n");
         return AVERROR_INVALIDDATA;
     }
@@ -335,7 +346,7 @@ static int xpm_decode_frame(AVCodecContext *avctx, void *data,
 
     size = 1;
     for (i = 0; i < cpp; i++)
-        size *= 94;
+        size *= 95;
 
     if (ncolors <= 0 || ncolors > size) {
         av_log(avctx, AV_LOG_ERROR, "invalid number of colors: %d\n", ncolors);
@@ -349,12 +360,15 @@ static int xpm_decode_frame(AVCodecContext *avctx, void *data,
         return AVERROR(ENOMEM);
 
     ptr += mod_strcspn(ptr, ",") + 1;
+    if (end - ptr < 1)
+        return AVERROR_INVALIDDATA;
+
     for (i = 0; i < ncolors; i++) {
         const uint8_t *index;
         int len;
 
         ptr += mod_strcspn(ptr, "\"") + 1;
-        if (ptr + cpp > end)
+        if (end - ptr < cpp)
             return AVERROR_INVALIDDATA;
         index = ptr;
         ptr += cpp;
@@ -373,14 +387,20 @@ static int xpm_decode_frame(AVCodecContext *avctx, void *data,
 
         x->pixels[ret] = color_string_to_rgba(ptr, len);
         ptr += mod_strcspn(ptr, ",") + 1;
+        if (end - ptr < 1)
+            return AVERROR_INVALIDDATA;
     }
 
     for (i = 0; i < avctx->height; i++) {
         dst = (uint32_t *)(p->data[0] + i * p->linesize[0]);
+        if (end - ptr < 1)
+            return AVERROR_INVALIDDATA;
         ptr += mod_strcspn(ptr, "\"") + 1;
+        if (end - ptr < 1)
+            return AVERROR_INVALIDDATA;
 
         for (j = 0; j < avctx->width; j++) {
-            if (ptr + cpp > end)
+            if (end - ptr < cpp)
                 return AVERROR_INVALIDDATA;
 
             if ((ret = ascii2index(ptr, cpp)) < 0)
@@ -405,6 +425,9 @@ static av_cold int xpm_decode_close(AVCodecContext *avctx)
     XPMDecContext *x = avctx->priv_data;
     av_freep(&x->pixels);
 
+    av_freep(&x->buf);
+    x->buf_size = 0;
+
     return 0;
 }
path: libavcodec/xpmdec.c