Repository: facebook/zstd

Problem statement:
ZSTD_decompress sometimes return corruption_detected instead of dstSize_tooSmall since 718f00ff6fe42db7e6ba09a7f7992b3e85283f77
**Describe the bug**
Since 718f00ff6fe42db7e6ba09a7f7992b3e85283f77, `ZSTD_decompress` for some buffer & size returns `ZSTD_error_corruption_detected` instead of `ZSTD_error_dstSize_tooSmall` depending on the size

**To Reproduce**
I have a reproducer but I can't share it as it contains PII :(
I'll happy to try any patch

```
#include <stdio.h>
#include <stddef.h>
#include <stdlib.h>
#include <stdint.h>
#include <zstd.h>
#include <zstd_errors.h>

#define NELEMS(x)  (sizeof(x) / sizeof((x)[0]))

int main(int argc, const char** argv)
{

    //Unable to begin ZSTD decompression (in buffer is 1670 bytes, out buffer is 10020 bytes): Corrupted block detected
    unsigned char buf1_bin[] = {
<can't share that>
};

    size_t ret;
    char *decompressed;
    decompressed = malloc(10*1024*1024);

    ret = ZSTD_decompress(decompressed, 3340, buf1_bin, NELEMS(buf1_bin));
    printf("%s\n", ZSTD_getErrorName(ret));

    ret = ZSTD_decompress(decompressed, 10020, buf1_bin, NELEMS(buf1_bin));
    printf("%s\n", ZSTD_getErrorName(ret));

    return ZSTD_getErrorCode(ret) != ZSTD_error_dstSize_tooSmall;
}
```

`git bisect run sh -c 'git clean -xfd && make lib && gcc ../bugzstd/bugzstd.c -o bugzstd -Ilib/ lib/libzstd.a && ./bugzstd'`
bisecting between v1.4.4 and v1.4.5 gave me 718f00ff6fe42db7e6ba09a7f7992b3e85283f77

This impacts librdkafka in https://github.com/edenhill/librdkafka/issues/2943
When `ZSTD_getFrameContentSize` returns `ZSTD_CONTENTSIZE_UNKNOWN`, librdkafka triple the dst buffer size until the decompression succeeds
Also librdkafka uses streaming compression but non streaming decompression
https://github.com/edenhill/librdkafka/blob/master/src/rdkafka_zstd.c

**Expected behavior**
Both size return "Destination buffer is too small"

**Actual Behavior**
After 718f00ff6fe42db7e6ba09a7f7992b3e85283f77, the second call returns "Corrupted block detected"

**Desktop (please complete the following information):**
 - OS: Fedora 32 & CentOS 7
 - Compiler: gcc
 - Flags: defaults
 - Build system: Makefile


Hints:
We will absolutely find and fix this issue. We don't have extensive tests to ensure that `ZSTD_error_dstSize_tooSmall ` always gets returned when the data is valid but the output buffer is too small, so that is something we will have to add.

The action items are:
1. Add a fuzzer in `tests/fuzzer` that generates compressed blocks and then decompresses them with an output buffer that is too small and ensures the result is `ZSTD_error_dstSize_tooSmall`.
2. Fix all issues found by the fuzzer.
2. Add unit tests for any issues found by the fuzzer.
I experience the same error after upgrading [gozstd](https://github.com/valyala/gozstd/) from `zstd` v1.4.4 to v1.4.5 
Below are samples of compressed blocks, which cannot be decompressed with zstd v1.4.5 when the destination buffer is too small:

```
block len=162;
block data (hex)=28B52FFD603701C50400F404000002E74EA2E80000000300BF0300000002000003021E000200A813031482060205017F1209090000000400000305060200DB12000000000100000303081E20060003061E4A8E001801000003031E220039012AE34066201B585B91805C02463300A064BB840B6A040A03400257806EC0770138E05ED00DF80ED80C1C0274081406DE4010A00F150698B05DA038F0005A59893508B0

block len=133;
block data(hex)=28B52FFD608801DD0300C4030000029DCD1DC0000000046C4909000000040000030618000102363D0303020022C606000000030300351C0F04C0DB05013F4907003C3D09040C6D051C0098AB10A01EF0DD8273C0770BCE01DF0150001D028505A500DDC0774B86D1A01EF82E15005EE50116C00AA00F150698B05DA038F0005A59893508B0

block len=140;
block data(hex)=28B52FFD6088011504001404000002E74EA2E80000000300BF0900000004000003061E000100B4F603000000020000030202A8130C03030301545604006B4D060285940801B899060904FCE2121D006016A01BF82E7D853820840BEA01DF2D38077CB7E01CF0DD8273E0BB5400441817C0BDA043A030000276C16AA10F150698B05DA038F0005A59893508B0

```

Patch:
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index be5c7cfc334..4b04218f7bf 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -579,11 +579,11 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
                           const void* src, size_t srcSize)
 {
     DEBUGLOG(5, "ZSTD_copyRawBlock");
+    RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
     if (dst == NULL) {
         if (srcSize == 0) return 0;
         RETURN_ERROR(dstBuffer_null, "");
     }
-    RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
     memcpy(dst, src, srcSize);
     return srcSize;
 }
@@ -592,11 +592,11 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
                                BYTE b,
                                size_t regenSize)
 {
+    RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
     if (dst == NULL) {
         if (regenSize == 0) return 0;
         RETURN_ERROR(dstBuffer_null, "");
     }
-    RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
     memset(dst, b, regenSize);
     return regenSize;
 }
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index e93d6febe93..ca0de662ec8 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -1084,14 +1084,14 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
 #endif
             DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
             BIT_reloadDStream(&(seqState.DStream));
+            op += oneSeqSize;
             /* gcc and clang both don't like early returns in this loop.
-             * gcc doesn't like early breaks either.
-             * Instead save an error and report it at the end.
-             * When there is an error, don't increment op, so we don't
-             * overwrite.
+             * Instead break and check for an error at the end of the loop.
              */
-            if (UNLIKELY(ZSTD_isError(oneSeqSize))) error = oneSeqSize;
-            else op += oneSeqSize;
+            if (UNLIKELY(ZSTD_isError(oneSeqSize))) {
+                error = oneSeqSize;
+                break;
+            }
             if (UNLIKELY(!--nbSeq)) break;
         }
 


Test patch:
diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile
index 1af3dc73419..42988c3478d 100644
--- a/tests/fuzz/Makefile
+++ b/tests/fuzz/Makefile
@@ -95,7 +95,8 @@ FUZZ_TARGETS :=       \
 	simple_compress \
 	dictionary_loader \
 	raw_dictionary_round_trip \
-	dictionary_stream_round_trip
+	dictionary_stream_round_trip \
+	decompress_dstSize_tooSmall
 
 all: $(FUZZ_TARGETS)
 
@@ -180,6 +181,9 @@ zstd_frame_info: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_zstd_frame_info.o
 dictionary_loader: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_loader.o
 	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_loader.o $(LIB_FUZZING_ENGINE) -o $@
 
+decompress_dstSize_tooSmall: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_decompress_dstSize_tooSmall.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_decompress_dstSize_tooSmall.o $(LIB_FUZZING_ENGINE) -o $@
+
 libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c d_fuzz_regression_driver.o
 	$(AR) $(FUZZ_ARFLAGS) $@ d_fuzz_regression_driver.o
 
diff --git a/tests/fuzz/decompress_dstSize_tooSmall.c b/tests/fuzz/decompress_dstSize_tooSmall.c
new file mode 100644
index 00000000000..e47b3d049a4
--- /dev/null
+++ b/tests/fuzz/decompress_dstSize_tooSmall.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2016-2020, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/**
+ * This fuzz target attempts to decompress a valid compressed frame into
+ * an output buffer that is too small to ensure we always get
+ * ZSTD_error_dstSize_tooSmall.
+ */
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include "fuzz_helpers.h"
+#include "zstd.h"
+#include "zstd_errors.h"
+#include "zstd_helpers.h"
+#include "fuzz_data_producer.h"
+
+static ZSTD_CCtx *cctx = NULL;
+static ZSTD_DCtx *dctx = NULL;
+
+int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
+{
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    size_t rBufSize = FUZZ_dataProducer_uint32Range(producer, 0, size);
+    size = FUZZ_dataProducer_remainingBytes(producer);
+    /* Ensure the round-trip buffer is too small. */
+    if (rBufSize >= size) {
+        rBufSize = size > 0 ? size - 1 : 0;
+    }
+    size_t const cBufSize = ZSTD_compressBound(size);
+
+    if (!cctx) {
+        cctx = ZSTD_createCCtx();
+        FUZZ_ASSERT(cctx);
+    }
+    if (!dctx) {
+        dctx = ZSTD_createDCtx();
+        FUZZ_ASSERT(dctx);
+    }
+
+    void *cBuf = FUZZ_malloc(cBufSize);
+    void *rBuf = FUZZ_malloc(rBufSize);
+    size_t const cSize = ZSTD_compressCCtx(cctx, cBuf, cBufSize, src, size, 1);
+    FUZZ_ZASSERT(cSize);
+    size_t const rSize = ZSTD_decompressDCtx(dctx, rBuf, rBufSize, cBuf, cSize);
+    if (size == 0) {
+        FUZZ_ASSERT(rSize == 0);
+    } else {
+        FUZZ_ASSERT(ZSTD_isError(rSize));
+        FUZZ_ASSERT(ZSTD_getErrorCode(rSize) == ZSTD_error_dstSize_tooSmall);
+    }
+    free(cBuf);
+    free(rBuf);
+    FUZZ_dataProducer_free(producer);
+#ifndef STATEFUL_FUZZING
+    ZSTD_freeCCtx(cctx); cctx = NULL;
+    ZSTD_freeDCtx(dctx); dctx = NULL;
+#endif
+    return 0;
+}
diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index 6875d1d6036..6332eeb9121 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -59,6 +59,7 @@ def __init__(self, input_type, frame_type=FrameType.ZSTD):
     'dictionary_loader': TargetInfo(InputType.DICTIONARY_DATA),
     'raw_dictionary_round_trip': TargetInfo(InputType.RAW_DATA),
     'dictionary_stream_round_trip': TargetInfo(InputType.RAW_DATA),
+    'decompress_dstSize_tooSmall': TargetInfo(InputType.RAW_DATA),
 }
 TARGETS = list(TARGET_INFO.keys())
 ALL_TARGETS = TARGETS + ['all']
diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c
index b64f373eb8a..620177fb0ef 100644
--- a/tests/fuzz/simple_compress.c
+++ b/tests/fuzz/simple_compress.c
@@ -19,6 +19,7 @@
 #include <stdio.h>
 #include "fuzz_helpers.h"
 #include "zstd.h"
+#include "zstd_errors.h"
 #include "zstd_helpers.h"
 #include "fuzz_data_producer.h"
 
@@ -42,7 +43,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     }
 
     void *rBuf = FUZZ_malloc(bufSize);
-    ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, cLevel);
+    size_t const ret = ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, cLevel);
+    if (ZSTD_isError(ret)) {
+        FUZZ_ASSERT(ZSTD_getErrorCode(ret) == ZSTD_error_dstSize_tooSmall);
+    }
     free(rBuf);
     FUZZ_dataProducer_free(producer);
 #ifndef STATEFUL_FUZZING
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 8ac2864f3e3..abb4e861890 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -3053,7 +3053,7 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const
                 DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n",
                             testNb, (unsigned)tooSmallSize, (unsigned)missing);
                 { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
-                  CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); }
+                  CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); }
                 { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck));
                   CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow  (check.%08X != %08X.mark)", endCheck, endMark); }
         }   }
@@ -3100,7 +3100,7 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const
             static const BYTE token = 0xA9;
             dstBuffer[tooSmallSize] = token;
             { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
-              CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); }
+              CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); }
             CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
         }