Repository: xroche/httrack (2026-07-27T11:50:38Z)

Issue:
A successful FTP transfer can be blanked when its backlog slot is swapped out
A completed FTP transfer can end as a 0-byte file while the engine records it as a success. `hts-cache/new.txt` says `4109/4109 ... 200 added ('OK')`, the server sent every byte and answered 226, and the file on disk is empty.

The FTP worker writes the body itself and leaves `r.is_write` at 0 with `r.adr == NULL` (`src/htsftp.c:630`). Once the slot turns `STATUS_READY`, `slot_can_be_cached_on_disk()` (`src/htsback.c:342`) accepts it, `back_cleanup_background()` serializes it into the on-disk ready table and calls `back_clear_entry()`. The slot that comes back has `size > 0`, no buffer and `is_write == 0`, so the later write of the link puts nothing over the bytes the worker had already produced.

It is a race against `back_clean()` landing between the slot going ready and the parser taking it, so it needs load. Three FTP URLs, same fixture, same machine (12 cores):

- default concurrency, idle: 0/25 runs lost a file
- default concurrency, 24 busy loops running: 7/15
- `-c1`, same load: 0/15

Reproducible with the FTP fixture added in #771 (`tests/ftp-server.py`): mirror three FTP files while the box is busy, then compare the mirrored sizes against the server root. `tests/97_local-ftp-refetch.test` pins `-c1` to stay clear of this.

Found while fixing #771 and independent of it: this hits a first crawl, where no re-fetch backup exists at all.


Discussion:

Patch:
diff --git a/src/htsback.c b/src/htsback.c
index d8ace590..d0fdbe79 100644
--- a/src/htsback.c
+++ b/src/htsback.c
@@ -391,6 +391,56 @@ int back_selftest_slot_swap(void) {
   CHECK(0, "the dummy test slot");
 #undef CHECK
 
+  /* The swap round-trip must not lose the size of a slot whose body is already
+     at url_sav, or the link writer blanks the file (#797). */
+  {
+    static const char body[] = "swapped body";
+    int c;
+
+    for (c = 0; c < 2; c++) {
+      const hts_boolean inmemory = c == 0 ? HTS_TRUE : HTS_FALSE;
+      FILE *const fp = tmpfile();
+      lien_back *copy = NULL;
+
+      memset(&back, 0, sizeof(back));
+      back.status = STATUS_READY;
+      strcpybuff(back.url_sav, "/tmp/httrack-selftest.bin");
+      back.r.size = (LLint) sizeof(body) - 1;
+      if (inmemory) {
+        back.r.adr = strdupt(body);
+      }
+      if (fp == NULL || back_serialize(fp, &back) != 0 ||
+          fseek(fp, 0, SEEK_SET) != 0 || back_unserialize(fp, &copy) != 0) {
+        fprintf(stderr, "backswap: round-trip failed for a %s slot\n",
+                inmemory ? "buffered" : "direct-to-disk");
+        err = 1;
+      } else {
+        if (copy->r.size != back.r.size) {
+          fprintf(stderr,
+                  "backswap: %s slot came back with size " LLintP
+                  ", expected " LLintP "\n",
+                  inmemory ? "buffered" : "direct-to-disk", copy->r.size,
+                  back.r.size);
+          err = 1;
+        }
+        if (inmemory && (copy->r.adr == NULL ||
+                         memcmp(copy->r.adr, body, sizeof(body) - 1) != 0)) {
+          fprintf(stderr, "backswap: buffered slot lost its body\n");
+          err = 1;
+        }
+        if (!inmemory && copy->r.adr != NULL) {
+          fprintf(stderr, "backswap: direct-to-disk slot gained a body\n");
+          err = 1;
+        }
+        back_clear_entry(copy);
+        freet(copy);
+      }
+      if (fp != NULL)
+        fclose(fp);
+      freet(back.r.adr);
+    }
+  }
+
   printf("backswap self-test: %s\n", err ? "FAIL" : "OK");
   return err;
 }
@@ -1266,7 +1316,10 @@ int back_unserialize(FILE * fp, lien_back ** dst) {
     (*dst)->r.ssl_con = NULL;
 #endif
     if (back_data_unserialize(fp, (void **) &(*dst)->r.adr, &size) == 0) {
-      (*dst)->r.size = size;
+      /* A bodyless slot already wrote its bytes to url_sav (FTP, direct to
+         disk); zeroing r.size makes the writer blank that file (#797). */
+      if ((*dst)->r.adr != NULL)
+        (*dst)->r.size = size;
       (*dst)->r.headers = NULL;
       if (back_string_unserialize(fp, &(*dst)->r.headers) == 0)
         return 0;               /* ok */


Test patch:
diff --git a/tests/110_local-ftp-parallel.test b/tests/110_local-ftp-parallel.test
new file mode 100644
index 00000000..30aec935
--- /dev/null
+++ b/tests/110_local-ftp-parallel.test
@@ -0,0 +1,98 @@
+#!/bin/bash
+#
+# A completed FTP transfer used to end as a 0-byte file when its ready backlog
+# slot was swapped to the on-disk table (#797): the FTP worker writes url_sav
+# itself and leaves no in-memory body, and the swap round-trip zeroed the slot's
+# size, so the link writer then created an empty file over the bytes.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+root="${tmpdir}/root"
+out="${tmpdir}/crawl"
+mkdir -p "$root" "$out"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+size_of() { wc -c <"$1" | tr -d '[:space:]'; }
+
+# Distinct bodies and lengths, so a mirror that mixed two of them up cannot
+# compare equal.
+names=""
+for i in 1 2 3 4 5 6 7 8 9 10 11 12; do
+    names="${names} f${i}"
+    "$python" -c 'import sys; sys.stdout.buffer.write(
+        ("BODY-%s " % sys.argv[1]).encode() + sys.argv[1].encode() * (400 * int(sys.argv[1])))' \
+        "$i" >"${root}/f${i}.bin"
+done
+
+serverlog="${tmpdir}/server.out"
+"$python" "$server" --root "$(nativepath "$root")" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 300); do
+    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "ftp server exited early: $(cat "$serverlog")" >&2
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
+    exit 1
+}
+host="127.0.0.1_${port}"
+
+urls=()
+for name in $names; do
+    urls+=("ftp://127.0.0.1:${port}/${name}.bin")
+done
+
+# -c8, never -c1: a slot is only swapped out while other transfers keep the
+# parser from picking it up. -Z because the swap only logs at debug level, and
+# the run below proves nothing unless it happened. Bounded like every crawl
+# pass: a wedge must fail the test, not hang the job until CI discards the log.
+run_with_timeout 300 httrack "${urls[@]}" -O "$out" --quiet -Z -c8 \
+    --disable-security-limits --robots=0 --timeout=20 --max-time=120 --retries=1 \
+    >"${tmpdir}/log" 2>&1
+
+swapped=$(grep -ac 'slots ready moved to background' "${out}/hts-log.txt" || true)
+test "${swapped:-0}" -gt 0 ||
+    fail "no ready slot was swapped to disk; this run never reached the bug"
+ok "the crawl swapped ready slots to the on-disk table ${swapped} times"
+
+for name in $names; do
+    test -f "${out}/${host}/${name}.bin" || fail "${name}.bin was not mirrored"
+    cmp -s "${out}/${host}/${name}.bin" "${root}/${name}.bin" ||
+        fail "${name}.bin has $(size_of "${out}/${host}/${name}.bin") of the" \
+            "$(size_of "${root}/${name}.bin") bytes served"
+done
+ok "every FTP transfer landed on disk intact"
+
+errors=$(grep -a 'Error:' "${out}/hts-log.txt" || true)
+test -z "$errors" || fail "the crawl reported errors: ${errors}"
+ok "no transfer was reported failed"
diff --git a/tests/Makefile.am b/tests/Makefile.am
index da28ebd8..6dda26ac 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -227,6 +227,7 @@ TESTS = \
 	103_teardown-status.test \
 	104_engine-warc-longurl.test \
 	105_suite-timeout.test \
+	110_local-ftp-parallel.test \
 	111_local-ftp-update-rest.test
 
 CLEANFILES = check-network_sh.cache