CVE-2017-11399: Out-of-bounds Read (CWE-125)
value: Integer overflow in the ape_decode_frame function in libavcodec/apedec.c in FFmpeg 2.4 through 3.3.2 allows remote attackers to cause a denial of service (out-of-array access and application crash) or possibly have unspecified other impact via a crafted APE file.
CWE description: The product reads data past the end, or before the beginning, of the intended buffer.
CVSS v3 base score: 7.8
Published: 2017-07-17T19:29Z
Repository: https://github.com/FFmpeg/FFmpeg
Language: C
Files changed: libavcodec/apedec.c
Commit date: 2017-07-16 16:53:02 +0200
Commit message: avcodec/apedec: Fix integer overflow

Fixes: out of array access
Fixes: PoC.ape and others

Found-by: Bingchang, Liu@VARAS of IIE
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

Diff: diff --git a/libavcodec/apedec.c b/libavcodec/apedec.c
index a6b14b8e247..15eb416ba4e 100644
--- a/libavcodec/apedec.c
+++ b/libavcodec/apedec.c
@@ -1410,10 +1410,11 @@ static int ape_decode_frame(AVCodecContext *avctx, void *data,
     uint8_t *sample8;
     int16_t *sample16;
     int32_t *sample24;
     int i, ch, ret;
     int blockstodecode;
+    uint64_t decoded_buffer_size;
 
     /* this should never be negative, but bad things will happen if it is, so
        check it just to make sure. */
     av_assert0(s->samples >= 0);
 
@@ -1465,11 +1466,11 @@ static int ape_decode_frame(AVCodecContext *avctx, void *data,
                 skip_bits_long(&s->gb, offset * 8);
             else
                 skip_bits_long(&s->gb, offset);
         }
 
-        if (!nblocks || nblocks > INT_MAX) {
+        if (!nblocks || nblocks > INT_MAX / 2 / sizeof(*s->decoded_buffer) - 8) {
             av_log(avctx, AV_LOG_ERROR, "Invalid sample count: %"PRIu32".\n",
                    nblocks);
             return AVERROR_INVALIDDATA;
         }
 
@@ -1491,12 +1492,13 @@ static int ape_decode_frame(AVCodecContext *avctx, void *data,
     // so we need to decode all of them at once
     if (s->fileversion < 3930)
         blockstodecode = s->samples;
 
     /* reallocate decoded sample buffer if needed */
-    av_fast_malloc(&s->decoded_buffer, &s->decoded_size,
-                   2 * FFALIGN(blockstodecode, 8) * sizeof(*s->decoded_buffer));
+    decoded_buffer_size = 2LL * FFALIGN(blockstodecode, 8) * sizeof(*s->decoded_buffer);
+    av_assert0(decoded_buffer_size <= INT_MAX);
+    av_fast_malloc(&s->decoded_buffer, &s->decoded_size, decoded_buffer_size);
     if (!s->decoded_buffer)
         return AVERROR(ENOMEM);
     memset(s->decoded_buffer, 0, s->decoded_size);
     s->decoded[0] = s->decoded_buffer;
     s->decoded[1] = s->decoded_buffer + FFALIGN(blockstodecode, 8);

Vulnerable code:         if (!nblocks || nblocks > INT_MAX) {
    av_fast_malloc(&s->decoded_buffer, &s->decoded_size,
                   2 * FFALIGN(blockstodecode, 8) * sizeof(*s->decoded_buffer));
Fixed code:     uint64_t decoded_buffer_size;
        if (!nblocks || nblocks > INT_MAX / 2 / sizeof(*s->decoded_buffer) - 8) {
    decoded_buffer_size = 2LL * FFALIGN(blockstodecode, 8) * sizeof(*s->decoded_buffer);
    av_assert0(decoded_buffer_size <= INT_MAX);
    av_fast_malloc(&s->decoded_buffer, &s->decoded_size, decoded_buffer_size);