CVE-2018-13304: Reachable Assertion (CWE-617)
value: In libavcodec in FFmpeg 4.0.1, improper maintenance of the consistency between the context profile field and studio_profile in libavcodec may trigger an assertion failure while converting a crafted AVI file to MPEG4, leading to a denial of service, related to error_resilience.c, h263dec.c, and mpeg4videodec.c.
CWE description: The product contains an assert() or similar statement that can be triggered by an attacker, which leads to an application exit or other behavior that is more severe than necessary.
CVSS v3 base score: 6.5
Published: 2018-07-05T17:29Z
Repository: https://github.com/FFmpeg/FFmpeg
Language: C
Files changed: libavcodec/error_resilience.c, libavcodec/h263dec.c, libavcodec/mpeg4videodec.c
Commit date: 2018-07-04 12:11:36 +0200
Commit message: avcodec/mpeg4videodec: Remove use of FF_PROFILE_MPEG4_SIMPLE_STUDIO as indicator of studio profile

The profile field is changed by code inside and outside the decoder,
its not a reliable indicator of the internal codec state.
Maintaining it consistency with studio_profile is messy.
Its easier to just avoid it and use only studio_profile

Fixes: assertion failure
Fixes: ffmpeg_crash_9.avi

Found-by: Thuan Pham, Marcel Böhme, Andrew Santosa and Alexandru Razvan Caciulescu with AFLSmart
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

Diff: diff --git a/libavcodec/error_resilience.c b/libavcodec/error_resilience.c
index 339042e2069f..1abae53f41d4 100644
--- a/libavcodec/error_resilience.c
+++ b/libavcodec/error_resilience.c
@@ -812,12 +812,11 @@ void ff_er_frame_start(ERContext *s)
 
 static int er_supported(ERContext *s)
 {
     if(s->avctx->hwaccel && s->avctx->hwaccel->decode_slice           ||
        !s->cur_pic.f                                                  ||
-       s->cur_pic.field_picture                                       ||
-       s->avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO
+       s->cur_pic.field_picture
     )
         return 0;
     return 1;
 }
 
diff --git a/libavcodec/h263dec.c b/libavcodec/h263dec.c
index f8a38083f23c..5967fca2858a 100644
--- a/libavcodec/h263dec.c
+++ b/libavcodec/h263dec.c
@@ -45,13 +45,14 @@
 #include "thread.h"
 #include "wmv2.h"
 
 static enum AVPixelFormat h263_get_format(AVCodecContext *avctx)
 {
+    MpegEncContext *s = avctx->priv_data;
     /* MPEG-4 Studio Profile only, not supported by hardware */
     if (avctx->bits_per_raw_sample > 8) {
-        av_assert1(avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO);
+        av_assert1(s->studio_profile);
         return avctx->pix_fmt;
     }
 
     if (avctx->codec->id == AV_CODEC_ID_MSS2)
         return AV_PIX_FMT_YUV420P;
@@ -668,11 +669,12 @@ retry:
             ff_msmpeg4_decode_ext_header(s, buf_size) < 0)
             s->er.error_status_table[s->mb_num - 1] = ER_MB_ERROR;
 
     av_assert1(s->bitstream_buffer_size == 0);
 frame_end:
-    ff_er_frame_end(&s->er);
+    if (!s->studio_profile)
+        ff_er_frame_end(&s->er);
 
     if (avctx->hwaccel) {
         ret = avctx->hwaccel->end_frame(avctx);
         if (ret < 0)
             return ret;
diff --git a/libavcodec/mpeg4videodec.c b/libavcodec/mpeg4videodec.c
index 54a84962444f..2df525e03a78 100644
--- a/libavcodec/mpeg4videodec.c
+++ b/libavcodec/mpeg4videodec.c
@@ -3242,11 +3242,10 @@ end:
     if (s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY)
         s->low_delay = 1;
     s->avctx->has_b_frames = !s->low_delay;
 
     if (s->studio_profile) {
-        av_assert0(s->avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO);
         if (!s->avctx->bits_per_raw_sample) {
             av_log(s->avctx, AV_LOG_ERROR, "Missing VOL header\n");
             return AVERROR_INVALIDDATA;
         }
         return decode_studio_vop_header(ctx, gb);

Vulnerable code:        s->cur_pic.field_picture                                       ||
       s->avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO
        av_assert1(avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO);
    ff_er_frame_end(&s->er);
        av_assert0(s->avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO);
Fixed code:        s->cur_pic.field_picture
    MpegEncContext *s = avctx->priv_data;
        av_assert1(s->studio_profile);
    if (!s->studio_profile)
        ff_er_frame_end(&s->er);