Repository: xroche/httrack (2026-07-15T06:54:50Z)

Issue:
--update: an incomplete transfer deletes the previously-mirrored file
When `--update` re-fetches an already-mirrored URL and the transfer comes up short (the server declares a `Content-Length` it does not deliver, or the connection drops mid-body), httrack correctly refuses to store the partial body and logs:

> incomplete transfer (expected 42 bytes, got 21): file not cached, will be retried on the next update

and then deletes the good local copy anyway.

The mirrored bytes themselves are never touched. The file dies in the end-of-session purge (`opt->delete_old`, on by default, `htscore.c:2081`): the purge removes every path listed in `hts-cache/old.lst` but absent from `hts-cache/new.lst`, and the incomplete-transfer branch in `back_finalize()` (`htsback.c:614-631`) returns early, before anything calls `filenote()` for that URL. Nothing ever writes it into `new.lst`, so the purge treats a file that failed to transfer exactly like a file that vanished from the site.

Reproduced on master (9ec5a3a) with a local server that serves a page in full on the first fetch and then, on the second, declares the same `Content-Length` while writing half the body:

1. Fresh crawl mirrors `trunc.html`.
2. `--update` re-crawl: the re-fetch is incomplete, httrack logs the message above, and the mirror keeps the good pass-1 bytes on disk.
3. End of session: `old.lst` lists `trunc.html`, `new.lst` does not, and the purge deletes it. Running the same update with `-X0` (purge off) keeps the file with its pass-1 content, which pins the mechanism.

This is not specific to compressed bodies (the plain-HTML repro above has no `Content-Encoding`), and it needs no abort, so it is broader than #521 and #557. It is the same data-loss family: a failed re-fetch must not destroy the copy it failed to refresh.

Fix shape: note the surviving copy before the early return, the way the decode-failure path in #560 now does.


Discussion:

Patch:
diff --git a/src/htsback.c b/src/htsback.c
index 6ddee3d9c..11ee4b2c0 100644
--- a/src/htsback.c
+++ b/src/htsback.c
@@ -628,6 +628,10 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                       back[p].r.size, back[p].url_adr, back[p].url_fil);
       }
       back_finalize_backup(opt, &back[p], HTS_FALSE);
+      /* Keep the surviving copy in new.lst, else the update purge drops the
+         file we refused to overwrite with the partial body (#562). */
+      if (fexist_utf8(back[p].url_sav))
+        filenote(&opt->state.strc, back[p].url_sav, NULL);
       return -1;
     }
 


Test patch:
diff --git a/tests/54_local-update-truncate-purge.test b/tests/54_local-update-truncate-purge.test
new file mode 100644
index 000000000..4b679f611
--- /dev/null
+++ b/tests/54_local-update-truncate-purge.test
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# An --update re-fetch that comes in short must not delete the mirrored file (#562).
+# Pass 1 mirrors page.html (in-memory) and file.bin (direct-to-disk); pass 2 serves
+# both with the full declared Content-Length but half the body, then closes. httrack
+# refuses the partial ("will be retried") — and unfixed, the end-of-update purge then
+# deletes the good pass-1 copy, which was never overwritten. stay.html is the control:
+# fully served both passes, it must update to V2.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --rerun-args '--update' \
+    --log-found 'incomplete transfer.*uptrunc/page\.html' \
+    --log-found 'incomplete transfer.*uptrunc/file\.bin' \
+    --file-matches 'uptrunc/page.html' 'MIRRORED-PAGE-V1' \
+    --file-matches 'uptrunc/file.bin' 'MIRRORED-BIN-V1' \
+    --file-min-bytes 'uptrunc/file.bin' 32768 \
+    --file-matches 'uptrunc/stay.html' 'STAY-V2' \
+    --file-not-matches 'uptrunc/stay.html' 'STAY-V1' \
+    httrack 'BASEURL/uptrunc/index.html'
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 419069596..1c7a7d8ff 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -130,6 +130,7 @@ TESTS = \
 	50_local-contentcodings.test \
 	51_local-update-codec.test \
 	52_local-socks5.test \
-	53_local-proxytrack-cache-corrupt.test
+	53_local-proxytrack-cache-corrupt.test \
+	54_local-update-truncate-purge.test
 
 CLEANFILES = check-network_sh.cache
diff --git a/tests/local-server.py b/tests/local-server.py
index 02a9aedc1..a7863d243 100755
--- a/tests/local-server.py
+++ b/tests/local-server.py
@@ -700,15 +700,17 @@ def route_codec_bin(self):
     # Pass 1 mirrors each file from a valid gzip body; pass 2 (--update) serves
     # a body that cannot be decoded. The previously-mirrored copy must survive.
     # fresh.html is the control: its pass-2 body decodes, so it must be updated.
-    UPCODEC_SEEN = {}
+    # Per-path body-fetch counter shared by the update-refetch routes; paths are
+    # distinct so one dict serves all of them.
+    REFETCH_SEEN = {}
 
-    def upcodec_pass(self):
-        """1 on the first body fetch of this path, 2 on the next ones. HEADs
-        don't count, so a stray one can't shift which pass gets the bad body."""
+    def refetch_pass(self):
+        """1 on the first body fetch of this path, N on the Nth. HEADs don't
+        count, so a stray one can't shift which pass gets the special body."""
         if self.command == "HEAD":
             return 1
-        seen = Handler.UPCODEC_SEEN.get(self.path, 0) + 1
-        Handler.UPCODEC_SEEN[self.path] = seen
+        seen = Handler.REFETCH_SEEN.get(self.path, 0) + 1
+        Handler.REFETCH_SEEN[self.path] = seen
         return seen
 
     @staticmethod
@@ -740,36 +742,78 @@ def route_upcodec_index(self):
     UNSUP_V1 = b"<html><body><p>MIRRORED-UNSUP-V1</p></body></html>"
 
     def route_upcodec_mem(self):
-        if self.upcodec_pass() == 1:
+        if self.refetch_pass() == 1:
             self.send_coded(self.gzipped(self.MEM_V1), "text/html")
         else:
             self.send_coded(self.bad_gzip(self.MEM_V1), "text/html")
 
     def route_upcodec_disk(self):
-        if self.upcodec_pass() == 1:
+        if self.refetch_pass() == 1:
             self.send_coded(self.gzipped(self.DISK_V1), "application/octet-stream")
         else:
             self.send_coded(self.bad_gzip(self.DISK_V1), "application/octet-stream")
 
     # Pass 2 switches to a coding we have no decoder for.
     def route_upcodec_unsup(self):
-        if self.upcodec_pass() == 1:
+        if self.refetch_pass() == 1:
             self.send_coded(self.gzipped(self.UNSUP_V1), "text/html")
         else:
             self.send_coded(self.UNSUP_V1, "text/html", coding="compress")
 
     def route_upcodec_fresh(self):
-        pass1 = self.upcodec_pass() == 1
+        pass1 = self.refetch_pass() == 1
         body = b"<html><body><p>FRESH-V%d</p></body></html>" % (1 if pass1 else 2)
         self.send_coded(self.gzipped(body), "text/html")
 
     # Same, direct-to-disk: the update pass decodes, so the temp is renamed over
     # an existing mirror file.
     def route_upcodec_freshdisk(self):
-        pass1 = self.upcodec_pass() == 1
+        pass1 = self.refetch_pass() == 1
         body = b"FRESHDISK-V%d\n" % (1 if pass1 else 2) + b"\x03\x02\x01\xfe" * 8192
         self.send_coded(self.gzipped(body), "application/octet-stream")
 
+    # #562: pass 1 mirrors fully; pass 2 (--update) declares the full
+    # Content-Length but delivers half then closes, so httrack refuses the partial.
+    PAGE_V1 = b"<html><body><p>MIRRORED-PAGE-V1</p></body></html>"
+    BIN_V1 = b"MIRRORED-BIN-V1\n" + b"\x00\x01\x02\xff" * 8192
+
+    def route_uptrunc_index(self):
+        self.send_html(
+            '\t<a href="page.html">page</a>\n'
+            '\t<a href="file.bin">file</a>\n'
+            '\t<a href="stay.html">stay</a>\n'
+        )
+
+    def send_truncated(self, body, content_type):
+        self.send_response(200)
+        self.send_header("Content-Type", content_type)
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            self.wfile.write(body[: len(body) // 2])  # short, then close
+            self.wfile.flush()
+        except OSError:
+            pass
+
+    def route_uptrunc_page(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.PAGE_V1, "text/html")
+        else:
+            self.send_truncated(self.PAGE_V1, "text/html")
+
+    def route_uptrunc_file(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.BIN_V1, "application/octet-stream")
+        else:
+            self.send_truncated(self.BIN_V1, "application/octet-stream")
+
+    # Control: fully served both passes, so a normal --update still lands.
+    def route_uptrunc_stay(self):
+        v = 1 if self.refetch_pass() == 1 else 2
+        self.send_raw(b"<html><body><p>STAY-V%d</p></body></html>" % v, "text/html")
+
     # Echo what httrack advertised, so a crawl can assert the header.
     def route_codec_ae(self):
         self.send_raw(
@@ -1473,6 +1517,10 @@ def route_maxrecv_404(self):
         "/upcodec/unsup.html": route_upcodec_unsup,
         "/upcodec/fresh.html": route_upcodec_fresh,
         "/upcodec/freshdisk.bin": route_upcodec_freshdisk,
+        "/uptrunc/index.html": route_uptrunc_index,
+        "/uptrunc/page.html": route_uptrunc_page,
+        "/uptrunc/file.bin": route_uptrunc_file,
+        "/uptrunc/stay.html": route_uptrunc_stay,
         "/types/index.html": route_types_index,
         "/types/control.php": route_types,
         "/types/photo.png": route_types,