Repository: xroche/httrack (2026-07-27T11:50:40Z)

Issue:
An FTP --update resumes a complete mirror with REST and splices the old file into the new body
run_launch_ftp() sends `REST <local size>` whenever the mirrored file merely exists (`src/htsftp.c:487`). On an `--update` pass every previously mirrored file exists, so a complete copy is treated as an interrupted download and the server is asked to continue it.

Against the fixture added in #771:

1. pass 1 mirrors a 4009-byte body starting `OLD-BODY`
2. the remote file is replaced by a different 6009-byte body starting `NEW-BODY`
3. pass 2 runs with `--update`

The mirrored file ends up 6009 bytes long, the first 4009 of them the old body and the last 2000 the tail of the new one. The length matches the remote's, so nothing downstream notices. The control, the same run with the server answering 500 to REST, produces the correct body and logs no error.

When the remote file keeps its length, REST lands at EOF, nothing arrives, and the stale copy is kept silently.

Separately, `r.size` counts only the bytes this transfer appended while `r.totalsize` holds the whole remote size, so the check at `src/htsftp.c:680` calls every resumed transfer "FTP file incomplete", including the no-op one above. That is an error line on every FTP `--update` of an unchanged file.

Resuming has to be gated on the local copy being a known-partial one, the way the HTTP side tracks it through the cache, with `r.size` seeded from the resume offset.


Discussion:

Patch:
diff --git a/src/htsftp.c b/src/htsftp.c
index f3c57e07..830abc12 100644
--- a/src/htsftp.c
+++ b/src/htsftp.c
@@ -483,16 +483,19 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                     back->r.totalsize = size;
                   }
                 }
-                // REST?
-                if (fexist(back->url_sav) && (transfer_list == 0)) {
+                /* Only over a copy back_add() judged partial: on --update every
+                   mirrored file exists, and resuming a complete one splices the
+                   old body into the new (#798). */
+                if (back->range_req_size > 0 && (transfer_list == 0)) {
                   strcpybuff(back->info, "rest");
-                  snprintf(line, sizeof(line), "REST " LLintP, (LLint) fsize(back->url_sav));
+                  snprintf(line, sizeof(line), "REST " LLintP,
+                           (LLint) back->range_req_size);
                   send_line(soc_ctl, line);
                   get_ftp_line(soc_ctl, line, sizeof(line), timeout);
                   _CHECK_HALT_FTP;
                   if ((line[0] == '3') || (line[0] == '2')) {   // ok
                     rest_understood = 1;
-                  }             // sinon tant pis 
+                  } // else never mind
                 }
               }                 // sinon tant pis 
             }
@@ -617,9 +620,12 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
         // Ok, connexion initiée
         //
         if (soc_dat != INVALID_SOCKET) {
-          if (rest_understood) {        // REST envoyée et comprise
+          if (rest_understood) { // REST sent and understood
             file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 0, 1,
                         0);
+            /* The bytes already on disk count too, or the completeness check
+               below rejects every resumed transfer (#798). */
+            back->r.size = back->range_req_size;
             back->r.fp = fileappend(&opt->state.strc, back->url_sav);
           } else {
             file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 1, 1,


Test patch:
diff --git a/tests/111_local-ftp-update-rest.test b/tests/111_local-ftp-update-rest.test
new file mode 100644
index 00000000..8a488b4c
--- /dev/null
+++ b/tests/111_local-ftp-update-rest.test
@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# FTP resumed with REST whenever the mirrored file merely existed, so an
+# --update over a complete copy spliced the old body into the new one and the
+# result matched the remote length (#798). Pass 1 is cut short to leave a real
+# partial, pass 2 must still resume it, pass 3 must not.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+root="${tmpdir}/root"
+out="${tmpdir}/crawl"
+mode="${tmpdir}/mode"
+cmds="${tmpdir}/cmds"
+mkdir -p "$root" "$out"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+size_of() { wc -c <"$1" | tr -d '[:space:]'; }
+# hts-log.txt is rewritten per pass, so this only ever sees the pass just run.
+no_errors() {
+    local errors
+    errors=$(grep -a 'Error:' "${out}/hts-log.txt" || true)
+    test -z "$errors" || fail "$1 reported errors: ${errors}"
+}
+# Commands the server saw since the last pass; each pass starts by emptying it.
+sent() { cat "$cmds"; }
+
+# The two generations differ in length and in their first bytes, so a splice
+# shows up whichever end of the file it lands in.
+write_body() {
+    "$python" -c 'import sys; sys.stdout.buffer.write(
+        ("%s-BODY " % sys.argv[1]).encode() + sys.argv[1][:1].encode() * int(sys.argv[2]))' \
+        "$1" "$2" >"${root}/a.bin"
+}
+write_body OLD 8000
+echo '/a.bin truncate' >"$mode"
+
+serverlog="${tmpdir}/server.out"
+"$python" "$server" --root "$(nativepath "$root")" \
+    --mode-file "$(nativepath "$mode")" --log "$(nativepath "$cmds")" \
+    >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 300); do
+    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "ftp server exited early: $(cat "$serverlog")" >&2
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
+    exit 1
+}
+host="127.0.0.1_${port}"
+url="ftp://127.0.0.1:${port}/a.bin"
+mirror="${out}/${host}/a.bin"
+# Bounded like every crawl pass: a wedge must fail the test, not hang the job.
+crawl() { run_with_timeout 300 httrack "$url" -O "$out" --quiet \
+    --disable-security-limits --robots=0 --timeout=20 --max-time=120 --retries=1 \
+    -c1 "$@"; }
+
+# --- pass 1: the transfer dies mid-body and leaves a partial -----------------
+crawl >"${tmpdir}/log1" 2>&1 || true
+test -f "$mirror" || fail "pass 1 mirrored nothing to resume"
+partial=$(size_of "$mirror")
+whole=$(size_of "${root}/a.bin")
+if test "$partial" -le 0 || test "$partial" -ge "$whole"; then
+    fail "pass 1 left ${partial} bytes, wanted a partial of ${whole}"
+fi
+ok "pass 1 left a ${partial}-byte partial copy"
+
+# --- pass 2: the partial is resumed ------------------------------------------
+: >"$mode"
+: >"$cmds"
+crawl >"${tmpdir}/log2" 2>&1
+case "$(sent)" in
+*"REST ${partial}"*) ok "pass 2 resumed at the partial's ${partial} bytes" ;;
+*) fail "pass 2 sent no REST ${partial}; commands were: $(sent)" ;;
+esac
+cmp -s "$mirror" "${root}/a.bin" ||
+    fail "the resumed copy is $(size_of "$mirror") bytes, served $(size_of "${root}/a.bin")"
+# A resume used to count only the appended tail against the remote size, so it
+# was reported "FTP file incomplete" even when it produced the whole file.
+no_errors "pass 2"
+ok "the resumed transfer produced the served body and no error"
+
+# --- pass 3: --update over that complete copy --------------------------------
+write_body NEW 12000
+: >"$cmds"
+crawl --update >"${tmpdir}/log3" 2>&1
+case "$(sent)" in
+*REST*) fail "the update resumed a complete mirror: $(sent)" ;;
+*RETR*) ok "the update re-fetched without REST" ;;
+*) fail "pass 3 never reached RETR; commands were: $(sent)" ;;
+esac
+cmp -s "$mirror" "${root}/a.bin" ||
+    fail "the updated copy is $(size_of "$mirror") bytes, served $(size_of "${root}/a.bin")"
+no_errors "pass 3"
+ok "the update replaced the mirror with the new body"
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 987f11e9..da28ebd8 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -226,6 +226,7 @@ TESTS = \
 	102_local-ftp-refetch.test \
 	103_teardown-status.test \
 	104_engine-warc-longurl.test \
-	105_suite-timeout.test
+	105_suite-timeout.test \
+	111_local-ftp-update-rest.test
 
 CLEANFILES = check-network_sh.cache