Repository: xroche/httrack (2026-07-27T19:30:13Z)

Issue:
A chunked response cut at a chunk boundary is stored and cached as complete
`back_finalize()`'s incomplete-transfer guard (`src/htsback.c`) is gated on `r.totalsize >= 0`, so it only fires for a response that declared a Content-Length. A chunked response has `totalsize == -1` and never reaches it. That is correct for the general case, but it means the engine cannot tell a complete chunked body from a truncated one, and it assumes complete.

Send `200` with `Transfer-Encoding: chunked`, write one data chunk and its trailing CRLF, then close the connection without the terminating zero-length chunk. `back_wait` sets `r.size == r.totalsize` at the chunk boundary, `READ_EOF` then sets `STATUS_READY`, and `statuscode` stays 200 because the error path only rewrites it when `statuscode <= 0`. `back_finalize` takes the success branch: the partial body is written to the mirror, cached, and on an `--update` it replaces whatever was there before.

For chunked this is detectable. The terminating chunk is part of the framing, so a stream that ended without it is truncated by definition, and the engine could refuse to store it the same way it refuses a short Content-Length body today.

An HTTP/1.0 identity body delimited only by connection close has no framing to check, so the same truncation is not knowable at the protocol level. A fix should say so rather than imply it covers that case too.

Found while fixing #775. That issue's guard makes the commit conditional on the new copy existing, which catches a `filecreate()` that failed outright. Existence is not completeness, so it does not catch this.



Discussion:

Patch:
diff --git a/src/htsback.c b/src/htsback.c
index a59072a0..097067a5 100644
--- a/src/htsback.c
+++ b/src/htsback.c
@@ -791,6 +791,12 @@ hts_boolean back_finalize_backup(httrackp *opt, lien_back *const back,
   return commit == wanted ? HTS_TRUE : HTS_FALSE;
 }
 
+/* A chunked body is framed by its terminating zero-length chunk (#840);
+   chunk_blocksize is reset per response and reaches -1 only once it is seen. */
+static hts_boolean back_chunked_unterminated(const lien_back *const back) {
+  return back->is_chunk && back->chunk_blocksize != -1 ? HTS_TRUE : HTS_FALSE;
+}
+
 // objet (lien) téléchargé ou transféré depuis le cache
 //
 // fermer les paramètres de transfert,
@@ -810,9 +816,22 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
     /* Don't store broken files. Note: check is done before compression.
        If the file is partial, the next run will attempt to continue it with compression too.
      */
-    if (back[p].r.totalsize >= 0 && back[p].r.statuscode > 0
-        && back[p].r.size != back[p].r.totalsize && !opt->tolerant) {
-      if (back[p].status == STATUS_READY) {
+    const hts_boolean cut_chunked = back_chunked_unterminated(&back[p]);
+    const hts_boolean short_body =
+        back[p].r.totalsize >= 0 && back[p].r.size != back[p].r.totalsize
+            ? HTS_TRUE
+            : HTS_FALSE;
+
+    if ((short_body || cut_chunked) && back[p].r.statuscode > 0 &&
+        !opt->tolerant) {
+      if (cut_chunked) {
+        hts_log_print(
+            opt, LOG_WARNING,
+            "truncated chunked transfer (terminating chunk missing, got " LLintP
+            " bytes): file not cached, will be retried on the next"
+            " update (use -%%B to cache anyway): %s%s",
+            back[p].r.size, back[p].url_adr, back[p].url_fil);
+      } else if (back[p].status == STATUS_READY) {
         hts_log_print(opt, LOG_WARNING,
                       "incomplete transfer (expected " LLintP
                       " bytes, got " LLintP
@@ -3536,6 +3555,26 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back_finalize(opt, cache, sback, i);
             }
 
+            /* Same treatment for an unterminated chunked stream: the byte count
+               agrees with the chunks that arrived, the framing does not. */
+            if (back[i].r.statuscode > 0 &&
+                back_chunked_unterminated(&back[i])) {
+              if (!opt->tolerant) {
+                deleteaddr(&back[i].r);
+                back[i].r.statuscode = STATUSCODE_CONNERROR; // recatch
+                htsblk_failf(&back[i].r,
+                             "Truncated chunked transfer (" LLintP
+                             " Bytes, terminating chunk missing)",
+                             (LLint) back[i].r.size);
+              } else {
+                hts_log_print(opt, LOG_WARNING,
+                              "Truncated chunked transfer (" LLintP
+                              " Bytes, terminating chunk missing) for %s%s",
+                              (LLint) back[i].r.size, back[i].url_adr,
+                              back[i].url_fil);
+              }
+            }
+
             if (back[i].r.totalsize >= 0) { // tester totalsize
               if (back[i].r.totalsize != back[i].r.size) {      // pas la même!
                 if (!opt->tolerant) {
@@ -3595,10 +3634,22 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                   if (back[i].r.totalsize < 0)
                     back[i].r.totalsize = 0;    // initialiser à 0 (-1 == unknown)
                   if (back[i].status == STATUS_CHUNK_WAIT) {    // "real" chunk
-                    if (sscanf(chunk_data, "%x", &chunk_size) == 1) {
+                    /* The chunk-size line is hostile input, so parse it wide
+                       and unsigned and drop anything an int cannot hold: sscanf
+                       "%x" lands 80000000 on INT_MIN, which sign-extends into a
+                       16EB realloc below and drives totalsize negative. */
+                    char *chunk_end = NULL;
+                    const unsigned long long chunk_value =
+                        strtoull(chunk_data, &chunk_end, 16);
+
+                    if (chunk_end != chunk_data && *chunk_end == '\0' &&
+                        chunk_value <= (unsigned long long) INT32_MAX) {
+                      chunk_size = (int) chunk_value;
                       if (chunk_size > 0)
                         back[i].chunk_blocksize = chunk_size;   /* the data block chunk size */
-                      else
+                      /* only a real 0 ends the stream; the bound above keeps a
+                         negative from ever claiming the sentinel (#840) */
+                      else if (chunk_size == 0)
                         back[i].chunk_blocksize = -1;   /* ending */
                       back[i].r.totalsize += chunk_size;        // noter taille
                       if (back[i].r.adr != NULL || !back[i].r.is_write) {       // Not to disk


Test patch:
diff --git a/tests/130_local-chunked-truncated.test b/tests/130_local-chunked-truncated.test
new file mode 100644
index 00000000..d5f7131d
--- /dev/null
+++ b/tests/130_local-chunked-truncated.test
@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# A chunked response cut before its terminating zero-length chunk must be refused
+# like a short Content-Length body (#840): never cached, and never allowed to
+# replace a good copy on an --update. As with a short Content-Length, a first
+# crawl still leaves the partial bytes of a direct-to-disk resource in the mirror
+# (always.bin below); only the in-memory HTML class never reaches the disk.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# First crawl: always.* are cut on every pass, the other three are the control.
+# reset.bin dies on an RST instead of a clean EOF, so the read error must keep
+# its own diagnostic rather than be relabelled by the framing check.
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-found 'truncated chunked transfer.*chunktrunc/always\.html' \
+    --log-found 'Truncated chunked transfer.*after 2 retries.*chunktrunc/always\.html' \
+    --log-found 'Interrupted transfer.*chunktrunc/reset\.bin' \
+    --log-not-found 'runcated chunked transfer.*chunktrunc/(page|stay|file|reset|hostile)' \
+    --not-found 'chunktrunc/always.html' \
+    --found 'chunktrunc/always.bin' \
+    --cache-not-found '/chunktrunc/always.html' \
+    --cache-not-found '/chunktrunc/always.bin' \
+    --cache-not-found '/chunktrunc/reset.bin' \
+    --cache-found '/chunktrunc/page.html' \
+    --cache-found '/chunktrunc/file.bin' \
+    --cache-found '/chunktrunc/hostile.html' \
+    --file-matches 'chunktrunc/page.html' 'CHUNKTRUNC-PAGE-V1' \
+    --file-matches 'chunktrunc/file.bin' 'CHUNKTRUNC-BIN-V1' \
+    httrack 'BASEURL/chunktrunc/index.html'
+
+# --update: page.html and file.bin come back cut, hostile.html comes back with a
+# 0x80000000 chunk-size line (rejected outright, since as an int it is INT_MIN
+# and would otherwise reach realloc sign-extended), stay.html is the control.
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --rerun-args '--update' \
+    --log-found 'truncated chunked transfer.*chunktrunc/page\.html' \
+    --log-found 'truncated chunked transfer.*chunktrunc/file\.bin' \
+    --log-found 'truncated chunked transfer.*chunktrunc/hostile\.html' \
+    --log-found 'Illegal chunk \(80000000\) for .*chunktrunc/hostile\.html' \
+    --log-not-found 'not enough memory \(-' \
+    --file-matches 'chunktrunc/page.html' 'CHUNKTRUNC-PAGE-V1' \
+    --file-not-matches 'chunktrunc/page.html' 'CHUNKTRUNC-PAGE-V2' \
+    --file-matches 'chunktrunc/file.bin' 'CHUNKTRUNC-BIN-V1' \
+    --file-not-matches 'chunktrunc/file.bin' 'CHUNKTRUNC-BIN-V2' \
+    --file-min-bytes 'chunktrunc/file.bin' 32768 \
+    --file-matches 'chunktrunc/hostile.html' 'CHUNKHOSTILE-V1' \
+    --cache-not-found '/chunktrunc/hostile.html' \
+    --file-matches 'chunktrunc/stay.html' 'CHUNKSTAY-V2' \
+    httrack 'BASEURL/chunktrunc/index.html'
+
+# -%B (tolerant) takes the truncated body, as it does a short Content-Length.
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-not-found 'truncated chunked transfer \(terminating chunk missing' \
+    --found 'chunktrunc/always.html' \
+    --cache-found '/chunktrunc/always.html' \
+    --cache-found '/chunktrunc/always.bin' \
+    httrack 'BASEURL/chunktrunc/index.html' '-%B'
diff --git a/tests/local-crawl.sh b/tests/local-crawl.sh
index 37e15f66..a90c4195 100755
--- a/tests/local-crawl.sh
+++ b/tests/local-crawl.sh
@@ -17,6 +17,7 @@
 #       --errors N --errors-content N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
+#       --cache-found URLTAIL ... --cache-not-found URLTAIL ... \
 #       --file-min-bytes PATH N --file-mode PATH OCTAL --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
 # --errors counts every "Error:" log line; --errors-content drops transient
@@ -25,6 +26,9 @@
 # --max/--min-mirror-bytes bound the mirrored content bytes (host root).
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
+# --cache-found/--cache-not-found assert whether hts-cache/new.zip holds an
+# entry whose URL ends with URLTAIL, e.g. /dir/page.html; being mirrored and
+# being cached are separate outcomes (#840).
 # --file-min-bytes asserts a mirrored file (PATH) is at least N bytes.
 # --file-mode asserts its octal permissions (e.g. 644); POSIX hosts only.
 # --rerun-args runs a second pass (same server and mirror dir) with the given
@@ -112,6 +116,25 @@ function find_hostroot {
     die "could not find host root under $out"
 }
 
+# Does the cache hold an entry whose URL ends with $1? An unreadable index is a
+# hard failure, else --cache-not-found would pass on a cache that never existed.
+# Suffix match over the whole key, so a URL with a query string needs the query
+# spelled out; a bare path would silently match nothing.
+function cache_has {
+    local rc
+    "$python" -c '
+import sys, zipfile
+try:
+    names = zipfile.ZipFile(sys.argv[1]).namelist()
+except Exception:
+    sys.exit(2)
+sys.exit(0 if any(n.endswith(sys.argv[2]) for n in names) else 1)
+' "${logroot}/hts-cache/new.zip" "$1"
+    rc=$?
+    test "$rc" -le 1 || die "cannot read cache index ${logroot}/hts-cache/new.zip"
+    return "$rc"
+}
+
 function assert_equals {
     info "$1"
     if test ! "$2" == "$3"; then
@@ -200,7 +223,7 @@ while test "$pos" -lt "$nargs"; do
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
-    --found | --not-found | --directory | --log-found | --log-not-found | --max-mirror-bytes | --min-mirror-bytes)
+    --found | --not-found | --directory | --log-found | --log-not-found | --max-mirror-bytes | --min-mirror-bytes | --cache-found | --cache-not-found)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
@@ -606,6 +629,22 @@ while test "$i" -lt "${#audit[@]}"; do
             exit 1
         fi
         ;;
+    --cache-found)
+        i=$((i + 1))
+        info "checking cache holds ${audit[$i]}"
+        if cache_has "${audit[$i]}"; then result "OK"; else
+            result "not cached"
+            exit 1
+        fi
+        ;;
+    --cache-not-found)
+        i=$((i + 1))
+        info "checking cache lacks ${audit[$i]}"
+        if cache_has "${audit[$i]}"; then
+            result "cached"
+            exit 1
+        else result "OK"; fi
+        ;;
     --found)
         i=$((i + 1))
         info "checking for ${audit[$i]}"
diff --git a/tests/local-server.py b/tests/local-server.py
index a173a917..f69404f1 100755
--- a/tests/local-server.py
+++ b/tests/local-server.py
@@ -19,6 +19,8 @@
 import hashlib
 import os
 import re
+import socket
+import struct
 import sys
 import time
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
@@ -1560,6 +1562,120 @@ def route_chunked_page(self):
             self.wfile.write(b"%X\r\n" % len(piece) + piece + b"\r\n")
         self.wfile.write(b"0\r\n\r\n")
 
+    # #840: a chunked stream cut before its terminating zero-length chunk.
+    CHUNKTRUNC_V1 = b"<html><body><p>CHUNKTRUNC-PAGE-V1</p></body></html>"
+    CHUNKTRUNC_V2 = b"<html><body><p>CHUNKTRUNC-PAGE-V2</p></body></html>"
+
+    CHUNKTRUNC_BIN_V1 = b"CHUNKTRUNC-BIN-V1\n" + b"\x07\x08\x09\xfe" * 8192
+    CHUNKTRUNC_BIN_V2 = b"CHUNKTRUNC-BIN-V2\n" + b"\x17\x18\x19\xee" * 8192
+
+    def route_chunktrunc_index(self):
+        self.send_html(
+            '\t<a href="page.html">page</a>\n'
+            '\t<a href="always.html">always</a>\n'
+            '\t<a href="stay.html">stay</a>\n'
+            '\t<a href="file.bin">file</a>\n'
+            '\t<a href="always.bin">alwaysbin</a>\n'
+            '\t<a href="hostile.html">hostile</a>\n'
+            '\t<a href="reset.bin">reset</a>\n'
+        )
+
+    def send_chunked(self, body, terminate, ctype="text/html; charset=utf-8"):
+        self.protocol_version = "HTTP/1.1"
+        self.send_response(200)
+        self.send_header("Content-Type", ctype)
+        self.send_header("Transfer-Encoding", "chunked")
+        self.send_header("Connection", "close")
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            self.wfile.write(b"%X\r\n" % len(body) + body + b"\r\n")
+            if terminate:
+                self.wfile.write(b"0\r\n\r\n")
+            self.wfile.flush()
+        except OSError:
+            pass
+        self.close_connection = True
+
+    def route_chunktrunc_page(self):
+        if self.refetch_pass() == 1:
+            self.send_chunked(self.CHUNKTRUNC_V1, True)
+        else:
+            self.send_chunked(self.CHUNKTRUNC_V2, False)
+
+    # Same over the direct-to-disk path: the update pass delivers half the new
+    # body and never terminates it.
+    def route_chunktrunc_file(self):
+        octet = "application/octet-stream"
+        if self.refetch_pass() == 1:
+            self.send_chunked(self.CHUNKTRUNC_BIN_V1, True, octet)
+        else:
+            half = self.CHUNKTRUNC_BIN_V2[: len(self.CHUNKTRUNC_BIN_V2) // 2]
+            self.send_chunked(half, False, octet)
+
+    # Truncated on every pass, so a first crawl has nothing good to fall back on.
+    def route_chunktrunc_always(self):
+        self.send_chunked(b"<html><body><p>CHUNKTRUNC-ALWAYS</p></body></html>", False)
+
+    def route_chunktrunc_alwaysbin(self):
+        self.send_chunked(
+            b"CHUNKTRUNC-ALWAYSBIN\n" + b"\x27\x28\x29\xde" * 1000,
+            False,
+            "application/octet-stream",
+        )
+
+    # Control: terminated on both passes, so a normal --update still lands.
+    def route_chunktrunc_stay(self):
+        v = 1 if self.refetch_pass() == 1 else 2
+        self.send_chunked(b"<html><body><p>CHUNKSTAY-V%d</p></body></html>" % v, True)
+
+    # The update pass declares a chunk of 0x80000000, which sscanf("%x") lands in
+    # an int as INT_MIN: it must not read as the terminating chunk, and the sum
+    # it drives negative must not read as a complete body either.
+    def route_chunktrunc_hostile(self):
+        if self.refetch_pass() == 1:
+            self.send_chunked(b"<html><body><p>CHUNKHOSTILE-V1</p></body></html>", True)
+            return
+        self.protocol_version = "HTTP/1.1"
+        self.send_response(200)
+        self.send_header("Content-Type", "text/html; charset=utf-8")
+        self.send_header("Transfer-Encoding", "chunked")
+        self.send_header("Connection", "close")
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            self.wfile.write(b"80000000\r\n")
+            self.wfile.flush()
+        except OSError:
+            pass
+        self.close_connection = True
+
+    # Aborts the chunked body with an RST, so the read fails rather than seeing a
+    # clean EOF and the transfer is already in error before the framing check.
+    def route_chunktrunc_reset(self):
+        self.protocol_version = "HTTP/1.1"
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Transfer-Encoding", "chunked")
+        self.send_header("Connection", "close")
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            body = b"CHUNKRESET\n" + b"\x31\x32\x33\xcd" * 4096
+            self.wfile.write(b"%X\r\n" % len(body) + body + b"\r\n")
+            self.wfile.flush()
+            time.sleep(0.5)  # let the client consume the chunk first
+            self.connection.setsockopt(
+                socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)
+            )
+            self.connection.close()
+        except OSError:
+            pass
+        self.close_connection = True
+
     # Content-Disposition naming: the attachment filename replaces the
     # URL-derived name; path components in it are stripped (RFC 2616).
     CDISPO_NAMES = {
@@ -1615,8 +1731,7 @@ def route_mini304_page(self):
 
     def route_bakname_index(self):
         self.send_html(
-            '\t<a href="a.bin">a</a>\n'
-            '\t<a href="hts-tmp/a.bin.bak">bak</a>\n'
+            '\t<a href="a.bin">a</a>\n' '\t<a href="hts-tmp/a.bin.bak">bak</a>\n'
         )
 
     def route_bakname_main(self):
@@ -2166,6 +2281,14 @@ def route_changes2_ticker(self):
         "/size/oversize.bin": route_size_oversize,
         "/chunked/index.html": route_chunked_index,
         "/chunked/page.html": route_chunked_page,
+        "/chunktrunc/index.html": route_chunktrunc_index,
+        "/chunktrunc/page.html": route_chunktrunc_page,
+        "/chunktrunc/always.html": route_chunktrunc_always,
+        "/chunktrunc/file.bin": route_chunktrunc_file,
+        "/chunktrunc/always.bin": route_chunktrunc_alwaysbin,
+        "/chunktrunc/stay.html": route_chunktrunc_stay,
+        "/chunktrunc/hostile.html": route_chunktrunc_hostile,
+        "/chunktrunc/reset.bin": route_chunktrunc_reset,
         "/errpage/index.html": route_errpage_index,
         "/errpage/good.html": route_errpage_good,
         "/errpage/missing.html": route_errpage_missing,
diff --git a/tests/tests-list.mk b/tests/tests-list.mk
index 96d04261..f8ee438c 100644
--- a/tests/tests-list.mk
+++ b/tests/tests-list.mk
@@ -201,6 +201,7 @@ TESTS += 108_engine-refetch-backup.test
 TESTS += 122_local-warc-revisit-headers.test
 TESTS += 119_local-proxytrack-ndx-fields2.test
 TESTS += 118_local-proxytrack-arcwrite.test
+TESTS += 130_local-chunked-truncated.test
 TESTS += 134_local-warc-revisit-ip.test
 TESTS += 135_local-warc-forced-notmodified.test
 TESTS += 136_local-warc-cache-priority.test