Repository: facebook/zstd

Problem statement:
zstd error out on skippable frames
Hi,
Zstd framing format is compatible with Lz4 for skippable frames (per @Cyan4973 and docs)
lz4 utility is skipping skippable frames, but zstd does not.
I created 48-byte file with one skippable frame:

 hexdump -C /tmp/test.zst
00000000  5e 2a 4d 18 28 00 00 00  00 00 00 00 00 00 00 00  |^*M.(...........|
00000010  00 00 00 00 16 03 01 00  c3 ff 0f 00 03 00 00 00  |................|
00000020  00 00 00 00 00 00 00 00  00 00 00 00 00 00 e3 ce  |................|
00000030

lz4 handles it:

lz4 -t /tmp/test.zst
/tmp/test.zst        : decoded 0 bytes  

zstd failing:

zstd -t /tmp/test.zst
/tmp/test.zst : Decoding error (36) : Corrupted block detected 

zstd tested on today's version of repo.

Please modify zstd utility to silently skip skippable frames (that is exactly what skippable frames for).
I am creating files with skippable frames that contains transport info, and I want to be able to decompress it using standard tools.

Also, I do have impression that originally zstd was handling skippable frames correctly - means silently skipping htem, like lz4? Am I wrong?

Thank you


Hints:
I've confirmed the problem, I'm looking into it.
Wow, that was quick. Thanks for looking.
I encounter other bug, when large block compressed in one shot gives an error when being decompressed by streaming decompress. Very weird error. I am trying to create test case.
I've bisected the bug to PR #836, but I don't know exactly what is going on yet.
That PR just exposed a bug that has been around since the beginning. Any skippable frame larger than the block size will be rejected. In the case of just a skippable frame the limit was 1024, and the PR reduced it to 4.

This patch will revert to the previous behavior, so if it worked before it should work with this patch, but I still need to fix the larger problem.

```patch
From 1b4d1b01ecd930e83d8ea263e85622635be53b07 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Wed, 1 Nov 2017 12:21:50 -0700
Subject: [PATCH] Revert to previous behavior

---
 lib/decompress/zstd_decompress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 96fc609..fdb65ef 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2502,7 +2502,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
 
             /* Adapt buffer sizes to frame header instructions */
-            {   size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
+            {   size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 1024 /* frame checksum */);
                 size_t const neededOutBuffSize = ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize);
                 if ((zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize)) {
                     size_t const bufferSize = neededInBuffSize + neededOutBuffSize;
-- 
2.10.0
```

Patch:
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 96fc6090896..caeafc255fe 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2555,17 +2555,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             /* fall-through */
         case zdss_load:
             {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
-                size_t const toLoad = neededInSize - zds->inPos;   /* should always be <= remaining space within inBuff */
+                size_t const toLoad = neededInSize - zds->inPos;
+                int const isSkipFrame = ZSTD_isSkipFrame(zds);
                 size_t loadedSize;
-                if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected);   /* should never happen */
-                loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
+                if (isSkipFrame) {
+                    loadedSize = MIN(toLoad, (size_t)(iend-ip));
+                } else {
+                    if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected);   /* should never happen */
+                    loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
+                }
                 ip += loadedSize;
                 zds->inPos += loadedSize;
                 if (loadedSize < toLoad) { someMoreWork = 0; break; }   /* not enough input, wait for more */
 
                 /* decode loaded input */
-                {  const int isSkipFrame = ZSTD_isSkipFrame(zds);
-                   size_t const decodedSize = ZSTD_decompressContinue(zds,
+                {   size_t const decodedSize = ZSTD_decompressContinue(zds,
                         zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
                         zds->inBuff, neededInSize);
                     if (ZSTD_isError(decodedSize)) return decodedSize;


Test patch:
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index e40f1997826..be9dc3e062e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -525,7 +525,7 @@ static int basicUnitTests(U32 seed, double compressibility)
             off += r;
             if (i == segs/2) {
                 /* insert skippable frame */
-                const U32 skipLen = 128 KB;
+                const U32 skipLen = 129 KB;
                 MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START);
                 MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen);
                 off += skipLen + ZSTD_skippableHeaderSize;
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 6f1f6df1856..53dbaf3bace 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -213,7 +213,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
 {
     size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
     void* CNBuffer = malloc(CNBufferSize);
-    size_t const skippableFrameSize = 11;
+    size_t const skippableFrameSize = 200 KB;
     size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
     void* compressedBuffer = malloc(compressedBufferSize);
     size_t const decodedBufferSize = CNBufferSize;