CVE-2018-12460: NULL Pointer Dereference (CWE-476)
value: libavcodec in FFmpeg 4.0 may trigger a NULL pointer dereference if the studio profile is incorrectly detected while converting a crafted AVI file to MPEG4, leading to a denial of service, related to idctdsp.c and mpegvideo.c.
CWE description: The product dereferences a pointer that it expects to be valid but is NULL.
CVSS v3 base score: 6.5
Published: 2018-06-15T15:29Z
Repository: https://github.com/FFmpeg/FFmpeg
Language: C
Files changed: libavcodec/idctdsp.c, libavcodec/idctdsp.h, libavcodec/mpegvideo.c
Commit date: 2018-05-30 17:58:08 +0200
Commit message: avcodec/idctdsp: Transmit studio_profile to init instead of using AVCodecContext profile

These 2 fields are not always the same, it is simpler to always use the same field
for detecting studio profile

Fixes: null pointer dereference
Fixes: ffmpeg_crash_3.avi

Found-by: Thuan Pham <thuanpv@comp.nus.edu.sg>, Marcel Böhme, Andrew Santosa and Alexandru RazvanCaciulescu with AFLSmart
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

Diff: diff --git a/libavcodec/idctdsp.c b/libavcodec/idctdsp.c
index 662033bd7807..954066a5e3c5 100644
--- a/libavcodec/idctdsp.c
+++ b/libavcodec/idctdsp.c
@@ -256,11 +256,11 @@ av_cold void ff_idctdsp_init(IDCTDSPContext *c, AVCodecContext *avctx)
         c->perm_type = FF_IDCT_PERM_NONE;
     } else {
         if (avctx->bits_per_raw_sample == 10 || avctx->bits_per_raw_sample == 9) {
             /* 10-bit MPEG-4 Simple Studio Profile requires a higher precision IDCT
                However, it only uses idct_put */
-            if (avctx->codec_id == AV_CODEC_ID_MPEG4 && avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO)
+            if (c->mpeg4_studio_profile)
                 c->idct_put              = ff_simple_idct_put_int32_10bit;
             else {
                 c->idct_put              = ff_simple_idct_put_int16_10bit;
                 c->idct_add              = ff_simple_idct_add_int16_10bit;
                 c->idct                  = ff_simple_idct_int16_10bit;
diff --git a/libavcodec/idctdsp.h b/libavcodec/idctdsp.h
index 26221f6a9dee..ca21a31a028c 100644
--- a/libavcodec/idctdsp.h
+++ b/libavcodec/idctdsp.h
@@ -93,10 +93,12 @@ typedef struct IDCTDSPContext {
      * - (-> decode coeffs -> zigzag reorder -> simple_mmx_perm -> dequant
      *    -> simple_idct_mmx -> ...)
      */
     uint8_t idct_permutation[64];
     enum idct_permutation_type perm_type;
+
+    int mpeg4_studio_profile;
 } IDCTDSPContext;
 
 void ff_put_pixels_clamped_c(const int16_t *block, uint8_t *av_restrict pixels,
                              ptrdiff_t line_size);
 void ff_add_pixels_clamped_c(const int16_t *block, uint8_t *av_restrict pixels,
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 45ea0f09e9cb..f75c0fd9b364 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -327,10 +327,12 @@ static av_cold int dct_init(MpegEncContext *s)
     return 0;
 }
 
 av_cold void ff_mpv_idct_init(MpegEncContext *s)
 {
+    if (s->codec_id == AV_CODEC_ID_MPEG4)
+        s->idsp.mpeg4_studio_profile = s->studio_profile;
     ff_idctdsp_init(&s->idsp, s->avctx);
 
     /* load & permutate scantables
      * note: only wmv uses different ones
      */

Vulnerable code:             if (avctx->codec_id == AV_CODEC_ID_MPEG4 && avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO)
Fixed code:             if (c->mpeg4_studio_profile)

    int mpeg4_studio_profile;
    if (s->codec_id == AV_CODEC_ID_MPEG4)
        s->idsp.mpeg4_studio_profile = s->studio_profile;