CVE-2023-46407: Out-of-bounds Read (CWE-125)
value: FFmpeg prior to commit bf814 was discovered to contain an out of bounds read via the dist->alphabet_size variable in the read_vlc_prefix() function.
CWE description: The product reads data past the end, or before the beginning, of the intended buffer.
CVSS v3 base score: 5.5
Published: 2023-10-27T20:15Z
Repository: https://github.com/FFmpeg/FFmpeg
Language: C
Files changed: libavcodec/jpegxl_parser.c
Commit date: 2023-10-17 08:40:49 -0400
Commit message: avcodec/jpegxl_parser: fix OOB read regression

In f7ac3512f5b5cb8eb149f37300b43461d8e93af3 the size of the dynamically
allocated buffer was shrunk, but it was made too small for very small
alphabet sizes. This patch restores the size to prevent an OOB read.

Reported-by: Cole Dilorenzo <coolkingcole@gmail.com>
Signed-off-by: Leo Izen <leo.izen@gmail.com>

Diff: diff --git a/libavcodec/jpegxl_parser.c b/libavcodec/jpegxl_parser.c
index dde36b0d6e62e..630fc8a60bff1 100644
--- a/libavcodec/jpegxl_parser.c
+++ b/libavcodec/jpegxl_parser.c
@@ -681,11 +681,11 @@ static int read_vlc_prefix(GetBitContext *gb, JXLEntropyDecoder *dec, JXLSymbolD
     uint32_t *level2_codecounts;
 
     int repeat_count_prev = 0, repeat_count_zero = 0, prev = 8;
     int total_code = 0, len, hskip, num_codes = 0, ret;
 
-    VLC level1_vlc;
+    VLC level1_vlc = { 0 };
 
     if (dist->alphabet_size == 1) {
         dist->vlc.bits = 0;
         dist->default_symbol = 0;
         return 0;
@@ -707,12 +707,14 @@ static int read_vlc_prefix(GetBitContext *gb, JXLEntropyDecoder *dec, JXLSymbolD
             level1_codecounts[0] += 18 - i - 1;
             break;
         }
     }
 
-    if (total_code != 32 && num_codes >= 2 || num_codes < 1)
-        return AVERROR_INVALIDDATA;
+    if (total_code != 32 && num_codes >= 2 || num_codes < 1) {
+        ret = AVERROR_INVALIDDATA;
+        goto end;
+    }
 
     for (int i = 1; i < 19; i++)
          level1_codecounts[i] += level1_codecounts[i - 1];
 
     for (int i = 17; i >= 0; i--) {
@@ -724,44 +726,46 @@ static int read_vlc_prefix(GetBitContext *gb, JXLEntropyDecoder *dec, JXLSymbolD
     ret = ff_vlc_init_from_lengths(&level1_vlc, 5, 18, level1_lens_s, 1, level1_syms, 2, 2,
         0, VLC_INIT_LE, dec->logctx);
     if (ret < 0)
         goto end;
 
-    buf = av_mallocz(dist->alphabet_size * (2 * sizeof(int8_t) + sizeof(int16_t) + sizeof(uint32_t))
+    buf = av_mallocz(MAX_PREFIX_ALPHABET_SIZE * (2 * sizeof(int8_t) + sizeof(int16_t) + sizeof(uint32_t))
                      + sizeof(uint32_t));
     if (!buf) {
         ret = AVERROR(ENOMEM);
         goto end;
     }
 
     level2_lens = (int8_t *)buf;
-    level2_lens_s = (int8_t *)(buf + dist->alphabet_size * sizeof(int8_t));
-    level2_syms = (int16_t *)(buf + dist->alphabet_size * (2 * sizeof(int8_t)));
-    level2_codecounts = (uint32_t *)(buf + dist->alphabet_size * (2 * sizeof(int8_t) + sizeof(int16_t)));
+    level2_lens_s = (int8_t *)(buf + MAX_PREFIX_ALPHABET_SIZE * sizeof(int8_t));
+    level2_syms = (int16_t *)(buf + MAX_PREFIX_ALPHABET_SIZE * (2 * sizeof(int8_t)));
+    level2_codecounts = (uint32_t *)(buf + MAX_PREFIX_ALPHABET_SIZE * (2 * sizeof(int8_t) + sizeof(int16_t)));
 
     total_code = 0;
     for (int i = 0; i < dist->alphabet_size; i++) {
         len = get_vlc2(gb, level1_vlc.table, 5, 1);
+        if (get_bits_left(gb) < 0) {
+            ret = AVERROR_BUFFER_TOO_SMALL;
+            goto end;
+        }
         if (len == 16) {
             int extra = 3 + get_bits(gb, 2);
             if (repeat_count_prev)
-                extra = 4 * (repeat_count_prev - 2) - repeat_count_prev + extra;
-            if (i + extra > dist->alphabet_size) {
-                ret = AVERROR_INVALIDDATA;
-                goto end;
-            }
+                extra += 4 * (repeat_count_prev - 2) - repeat_count_prev;
+            extra = FFMIN(extra, dist->alphabet_size - i);
             for (int j = 0; j < extra; j++)
                 level2_lens[i + j] = prev;
             total_code += (32768 >> prev) * extra;
             i += extra - 1;
             repeat_count_prev += extra;
             repeat_count_zero = 0;
             level2_codecounts[prev] += extra;
         } else if (len == 17) {
             int extra = 3 + get_bits(gb, 3);
             if (repeat_count_zero > 0)
-                extra = 8 * (repeat_count_zero - 2) - repeat_count_zero + extra;
+                extra += 8 * (repeat_count_zero - 2) - repeat_count_zero;
+            extra = FFMIN(extra, dist->alphabet_size - i);
             i += extra - 1;
             repeat_count_prev = 0;
             repeat_count_zero += extra;
             level2_codecounts[0] += extra;
         } else {

Vulnerable code:     VLC level1_vlc;
    if (total_code != 32 && num_codes >= 2 || num_codes < 1)
        return AVERROR_INVALIDDATA;
    buf = av_mallocz(dist->alphabet_size * (2 * sizeof(int8_t) + sizeof(int16_t) + sizeof(uint32_t))
    level2_lens_s = (int8_t *)(buf + dist->alphabet_size * sizeof(int8_t));
    level2_syms = (int16_t *)(buf + dist->alphabet_size * (2 * sizeof(int8_t)));
    level2_codecounts = (uint32_t *)(buf + dist->alphabet_size * (2 * sizeof(int8_t) + sizeof(int16_t)));
                extra = 4 * (repeat_count_prev - 2) - repeat_count_prev + extra;
            if (i + extra > dist->alphabet_size) {
                ret = AVERROR_INVALIDDATA;
                goto end;
            }
                extra = 8 * (repeat_count_zero - 2) - repeat_count_zero + extra;
Fixed code:     VLC level1_vlc = { 0 };
    if (total_code != 32 && num_codes >= 2 || num_codes < 1) {
        ret = AVERROR_INVALIDDATA;
        goto end;
    }
    buf = av_mallocz(MAX_PREFIX_ALPHABET_SIZE * (2 * sizeof(int8_t) + sizeof(int16_t) + sizeof(uint32_t))
    level2_lens_s = (int8_t *)(buf + MAX_PREFIX_ALPHABET_SIZE * sizeof(int8_t));
    level2_syms = (int16_t *)(buf + MAX_PREFIX_ALPHABET_SIZE * (2 * sizeof(int8_t)));
    level2_codecounts = (uint32_t *)(buf + MAX_PREFIX_ALPHABET_SIZE * (2 * sizeof(int8_t) + sizeof(int16_t)));
        if (get_bits_left(gb) < 0) {
            ret = AVERROR_BUFFER_TOO_SMALL;
            goto end;
        }
                extra += 4 * (repeat_count_prev - 2) - repeat_count_prev;
            extra = FFMIN(extra, dist->alphabet_size - i);
                extra += 8 * (repeat_count_zero - 2) - repeat_count_zero;
            extra = FFMIN(extra, dist->alphabet_size - i);