Repository: xroche/httrack (2026-07-27T07:00:20Z)

Issue:
A re-fetch that never gets a response truncates the mirrored file to zero on macOS
On the macOS CI runner, a re-fetch whose connection dies before the status line arrives overwrites the mirrored file with zero bytes. The same crawl on Linux leaves the previous copy alone.

Seen in run 30220185765 (job "build (macOS arm64, clang)"), on a fixture that serves `/changes/reset.bin` in full on the first fetch and, on every fetch after that, closes the connection without answering. The change report for pass 2 puts the file under `changed` with:

```
{ "url": ".../changes/reset.bin", "file": "127.0.0.1_50718/changes/reset.bin", "size": 0, "previous_size": 2062 }
```

`size` is a plain `fsize_utf8()` of the path after the crawl, so the 2062 bytes pass 1 mirrored are gone and a zero-length file is in their place. Three attempts (`--retries=2`) all failed, so nothing was ever received to write.

The suspect is the save gate in `htscore.c` (around the "An empty body must not overwrite the file when the transfer failed" comment): it lets a zero-length body through on `r.size == 0 && r.statuscode > 0`. If the Darwin socket path leaves a positive status code behind on a connection that produced no response, `filesave()` runs and truncates. I have not reproduced it locally, so which of the two differs, the status code or the body pointer, is still open.

Related to #746 and to the #562 family: a failed re-fetch must not damage the copy it failed to refresh. This one destroys the bytes directly rather than through the purge.



Discussion:

Patch:
diff --git a/src/htscore.c b/src/htscore.c
index a904c6fe..1ef711ee 100644
--- a/src/htscore.c
+++ b/src/htscore.c
@@ -1980,10 +1980,9 @@ int httpmirror(char *url1, httrackp * opt) {
           }
 
           // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
-          // An empty body must not overwrite the file when the transfer failed
-          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
-          // copy to 0 (#77 follow-up).
-          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
+          // A failed transfer has no body: r.adr holds debris from the aborted
+          // read, which would destroy the copy being re-fetched (#748).
+          if (r.statuscode > 0 && (r.adr != NULL || r.size == 0)) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {


Test patch:
diff --git a/tests/93_local-changes.test b/tests/93_local-changes.test
index ba465bb3..63a6ed23 100644
--- a/tests/93_local-changes.test
+++ b/tests/93_local-changes.test
@@ -94,10 +94,7 @@ expect_counts_match() {
     echo "OK"
 }
 lines() { printf '%s\n' "$@" | sort; }
-# A connection killed before the status line surfaces differently per platform
-# (macOS truncates the file, #748; Linux leaves it), so reset.bin is asserted on
-# its own and kept out of the exact lists.
-listed_but_reset() { listed "$1" | grep -v '/reset\.bin$' | sort; }
+digest() { cksum <"$1" | tr -d '[:space:]'; }
 
 # --- pass 1: nothing to compare against --------------------------------------
 httrack "${common[@]}" -O "$out" --purge-old=0 "${base}/changes/index.html" \
@@ -120,6 +117,7 @@ grep -aq "first crawl" "${out}/hts-log.txt" || {
 }
 
 host="127.0.0.1_${port}"
+resetdigest=$(digest "${out}/${host}/changes/reset.bin")
 # A leftover from some earlier failed attempt, at a name pass 2 fetches for the
 # first time: on disk, but never part of the previous mirror, so it is new.
 printf 'leftover junk' >"${out}/${host}/changes/fresh.html"
@@ -144,21 +142,19 @@ expect "gone is doomed.html and the old redirect target" \
 expect "changed is exactly the four moved resources" \
     "$(lines "${host}/changes/coded.bin" "${host}/changes/index.html" \
         "${host}/changes/moved.bin" "${host}/changes/moved.html")" \
-    "$(listed_but_reset changed)"
+    "$(listed changed | sort)"
 # Re-served byte for byte, 200, no Last-Modified, no ETag. flaky.bin gets there
 # through a failed transfer and a retry, so its file is notified twice;
 # redirtarget.html arrives behind a 302. sized.html has a byte-identical
 # payload, but the rewritten link to the renamed redirect target makes the file
-# on disk change length.
-expect "unchanged is exactly the six stable resources" \
+# on disk change length. reset.bin never completes a transfer (#746), so its
+# previous copy stands.
+expect "unchanged is exactly the seven stable resources" \
     "$(lines "${host}/changes/codedstable.bin" "${host}/changes/flaky.bin" \
-        "${host}/changes/redirtarget.html" "${host}/changes/sized.html" \
-        "${host}/changes/stable.bin" "${host}/changes/stable.html")" \
-    "$(listed_but_reset unchanged)"
-# reset.bin never completes a transfer, so it drops out of new.lst with its
-# mirrored copy still there. Whatever else it is, it is not a deletion.
-expect "a failed re-fetch is never reported gone" "" \
-    "$(listed gone | grep '/reset\.bin$' || true)"
+        "${host}/changes/redirtarget.html" "${host}/changes/reset.bin" \
+        "${host}/changes/sized.html" "${host}/changes/stable.bin" \
+        "${host}/changes/stable.html")" \
+    "$(listed unchanged | sort)"
 expect_counts_match "pass 2 counts match its lists"
 
 # Every mirrored file appears once and only once, across all four buckets.
@@ -192,19 +188,15 @@ test -f "${out}/${host}/changes/doomed.html" || {
     exit 1
 }
 echo "OK"
-printf '[a failed transfer keeps its file] ..\t'
-test -f "${out}/${host}/changes/reset.bin" || {
-    echo "FAIL: reset.bin lost its previous copy"
-    exit 1
-}
-echo "OK"
+expect "a failed transfer keeps its bytes" "$resetdigest" \
+    "$(digest "${out}/${host}/changes/reset.bin")"
 
 # --- pass 3: the same run with purging on ------------------------------------
 httrack "${common[@]}" -O "$out" --update "${base}/changes/index.html" \
     >"${tmpdir}/log3" 2>&1
 expect "the report says files were purged" "true" "$(field purged)"
 expect "gone is the page that only pass 2 linked" \
-    "${host}/changes/transient.html" "$(listed_but_reset gone)"
+    "${host}/changes/transient.html" "$(listed gone | sort)"
 printf '[a purged file is off disk] ..\t'
 test ! -f "${out}/${host}/changes/transient.html" || {
     echo "FAIL: transient.html survived --purge-old"
diff --git a/tests/96_local-refetch-keep.test b/tests/96_local-refetch-keep.test
new file mode 100644
index 00000000..02bba14d
--- /dev/null
+++ b/tests/96_local-refetch-keep.test
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# A re-fetch cut mid-header must leave the mirrored file alone (#748): unfixed,
+# the aborted read's leftover buffer lands on it. err.bin is the control, an
+# HTTP 500 on the same resource already keeping its copy; stay.bin answers
+# normally with a new body, so a fix that stopped overwriting anything fails.
+#
+# Purging is off: an update purge deletes both files for an unrelated reason
+# (#746), which would hide what this asserts.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --rerun-args '--update --purge-old=0' \
+    --log-found 'after 2 retries at link .*keep/page\.html' \
+    --log-found 'after 2 retries at link .*keep/data\.bin' \
+    --file-matches 'keep/page.html' 'KEEP-PAGE-V1' \
+    --file-not-matches 'keep/page.html' 'HTTP/1\.0 200' \
+    --file-matches 'keep/data.bin' 'KEEP-BIN-V1' \
+    --file-not-matches 'keep/data.bin' 'HTTP/1\.0 200' \
+    --file-min-bytes 'keep/data.bin' 2048 \
+    --file-matches 'keep/err.bin' 'KEEP-ERR-V1' \
+    --file-min-bytes 'keep/err.bin' 2048 \
+    --file-matches 'keep/stay.bin' 'KEEP-STAY-V2' \
+    --file-min-bytes 'keep/stay.bin' 2048 \
+    httrack 'BASEURL/keep/index.html' --retries=2
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 0d9a1e84..d2603648 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -194,6 +194,7 @@ TESTS = \
 	92_local-proxytrack-ndx-fields.test \
 	93_local-changes.test \
 	94_local-single-file.test \
-	95_local-sitemap.test
+	95_local-sitemap.test \
+	96_local-refetch-keep.test
 
 CLEANFILES = check-network_sh.cache
diff --git a/tests/local-server.py b/tests/local-server.py
index 72051663..9d6e1752 100755
--- a/tests/local-server.py
+++ b/tests/local-server.py
@@ -1000,6 +1000,62 @@ def route_uptrunc_stay(self):
         v = 1 if self.refetch_pass() == 1 else 2
         self.send_raw(b"<html><body><p>STAY-V%d</p></body></html>" % v, "text/html")
 
+    # --- re-fetch cut mid-header, so nothing is stored (#746, #748) ---------
+    KEEP_PAGE = b"<html><body><p>KEEP-PAGE-V1</p></body></html>"
+    KEEP_BIN = b"KEEP-BIN-V1\n" + b"\x51\x52\x53\x54" * 512
+    KEEP_ERR = b"KEEP-ERR-V1\n" + b"\x61\x62\x63\x64" * 512
+
+    def send_cut_headers(self):
+        """Hang up mid-header: the response never becomes parseable."""
+        self.close_connection = True
+        try:
+            self.wfile.write(b"HTTP/1.0 200 OK\r\nContent-Ty")
+            self.wfile.flush()
+        except OSError:
+            pass
+        self.connection.close()
+
+    def route_keep_index(self):
+        self.refetch_pass()
+        self.send_html(
+            '\t<a href="page.html">page</a>\n'
+            '\t<a href="data.bin">data</a>\n'
+            '\t<a href="err.bin">err</a>\n'
+            '\t<a href="stay.bin">stay</a>\n'
+        )
+
+    def route_keep_page(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.KEEP_PAGE, "text/html")
+        else:
+            self.send_cut_headers()
+
+    def route_keep_data(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.KEEP_BIN, "application/octet-stream")
+        else:
+            self.send_cut_headers()
+
+    # Control: an HTTP error on the same resource already keeps the copy, so
+    # the cut-header routes above must end up indistinguishable from it.
+    def route_keep_err(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.KEEP_ERR, "application/octet-stream")
+        else:
+            self.send_response(500)
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Length", "0")
+            self.end_headers()
+
+    # Control: answers normally, with a new body on pass 2, so a fix that
+    # stopped overwriting mirrored files altogether would be caught.
+    def route_keep_stay(self):
+        v = 1 if self.refetch_pass() == 1 else 2
+        self.send_raw(
+            b"KEEP-STAY-V%d\n" % v + b"\x71\x72\x73\x74" * 512,
+            "application/octet-stream",
+        )
+
     # Echo what httrack advertised, so a crawl can assert the header.
     def route_codec_ae(self):
         self.send_raw(
@@ -1950,6 +2006,11 @@ def route_changes2_ticker(self):
         "/uptrunc/page.html": route_uptrunc_page,
         "/uptrunc/file.bin": route_uptrunc_file,
         "/uptrunc/stay.html": route_uptrunc_stay,
+        "/keep/index.html": route_keep_index,
+        "/keep/page.html": route_keep_page,
+        "/keep/data.bin": route_keep_data,
+        "/keep/err.bin": route_keep_err,
+        "/keep/stay.bin": route_keep_stay,
         "/types/index.html": route_types_index,
         "/types/control.php": route_types,
         "/types/photo.png": route_types,