Repository: xroche/httrack (2026-07-27T12:52:47Z)

Issue:
An FTP resume can splice a changed remote when there is no cache entry
When httrack holds a local FTP file but no cache entry for it (`--cache=0`, or an `hts-cache` that was deleted), `back_add()` takes `range_req_size` from the file size at src/htsback.c:2074 and the transfer resumes with `REST`. If the remote changed since that local copy was written, the server appends the new tail onto the old head, and the mirrored file ends up as neither version.

PR #810 fixes the neighbouring case, a complete mirror resumed under `--update`, but leaves this one alone on purpose. The cache entry is what distinguishes a stale local copy from a genuine partial, so with no cache there is nothing to check against. HTTP does not have the problem because the server can reject a conditional request with 412 or 416. FTP has no conditional retrieval, so the client has to make the call itself.

This is not a regression. Before #810 the same splice was reachable through the `fexist()` test. Found while reviewing #810.


Discussion:

Patch:
diff --git a/src/htsftp.c b/src/htsftp.c
index 830abc12..14572c29 100644
--- a/src/htsftp.c
+++ b/src/htsftp.c
@@ -149,6 +149,61 @@ void ftp_split_userpass(const char *src, const char *end, char *user,
   }
 }
 
+/* Build "<verb> <path>", quoting a path the server could not parse bare. */
+static void ftp_command(char *line, size_t line_size, const char *verb,
+                        const char *path) {
+  if (strchr(path, ' ') != NULL || strchr(path, '\"') != NULL ||
+      strchr(path, '\'') != NULL)
+    snprintf(line, line_size, "%s \"%s\"", verb, path);
+  else
+    snprintf(line, line_size, "%s %s", verb, path);
+}
+
+/* MDTM reply "213 YYYYMMDDHHMMSS[.frac]" (RFC 3659, UTC) into tm_time. */
+static hts_boolean ftp_parse_mdtm(const char *line, struct tm *tm_time) {
+  int year, mon, mday, hour, min, sec;
+
+  if (sscanf(line, "213 %4d%2d%2d%2d%2d%2d", &year, &mon, &mday, &hour, &min,
+             &sec) != 6)
+    return HTS_FALSE;
+  if (year < 1900 || mon < 1 || mon > 12 || mday < 1 || mday > 31 || hour < 0 ||
+      hour > 23 || min < 0 || min > 59 || sec < 0 || sec > 60)
+    return HTS_FALSE;
+  memset(tm_time, 0, sizeof(*tm_time));
+  tm_time->tm_year = year - 1900;
+  tm_time->tm_mon = mon - 1;
+  tm_time->tm_mday = mday;
+  tm_time->tm_hour = hour;
+  tm_time->tm_min = min;
+  tm_time->tm_sec = sec;
+  tm_time->tm_isdst = 0;
+  return HTS_TRUE;
+}
+
+/* Whether the local copy can still be a prefix of the remote file. FTP has no
+   conditional fetch, so an unprovable answer has to be no (#823). */
+static hts_boolean ftp_may_resume(httrackp *opt, const lien_back *back,
+                                  time_t remote_mtime) {
+  const time_t local_mtime = get_filetime(back->url_sav);
+
+  if (back->r.totalsize <= back->range_req_size) {
+    hts_log_print(opt, LOG_DEBUG,
+                  "FTP: not resuming %s%s, the local copy is not shorter",
+                  back->url_adr, back->url_fil);
+    return HTS_FALSE;
+  }
+  /* Equality, not ordering: a copy this code did not stamp carries a local
+     clock time newer than any MDTM, which "not newer" would wave through. */
+  if (remote_mtime == (time_t) -1 || local_mtime == (time_t) -1 ||
+      remote_mtime != local_mtime) {
+    hts_log_print(opt, LOG_DEBUG,
+                  "FTP: not resuming %s%s, the copy is not dated as the remote",
+                  back->url_adr, back->url_fil);
+    return HTS_FALSE;
+  }
+  return HTS_TRUE;
+}
+
 // la véritable fonction une fois lancées les routines thread/fork
 int run_launch_ftp(FTPDownloadStruct * pStruct) {
   lien_back *back = pStruct->pBack;
@@ -184,6 +239,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   strcpybuff(back->r.msg, "");
   back->r.statuscode = 0;
   back->r.size = 0;
+  back->r.lastmodified[0] = '\0'; // a retry must not stamp the previous MDTM
 
   // récupérer user et pass si présents, et sauter user:id@ dans adr
   real_adr = strchr(back->url_adr, ':');
@@ -458,14 +514,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               char catbuff[CATBUFF_SIZE];
               char *ua = unescape_http(catbuff, sizeof(catbuff), ftp_filename);
 
-              if ((strchr(ua, ' '))
-                  || (strchr(ua, '\"'))
-                  || (strchr(ua, '\''))
-                ) {
-                snprintf(line, sizeof(line), "SIZE \"%s\"", ua);
-              } else {
-                snprintf(line, sizeof(line), "SIZE %s", ua);
-              }
+              ftp_command(line, sizeof(line), "SIZE", ua);
 
               // SIZE?
               strcpybuff(back->info, "size");
@@ -474,6 +523,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               _CHECK_HALT_FTP;
               if (line[0] == '2') {     // SIZE compris, ALORS tester REST (sinon pas tester: cf probleme des txt.gz decompresses a la volee)
                 char *szstr = strchr(line, ' ');
+                time_t remote_mtime = (time_t) -1;
+                struct tm remote_tm;
 
                 if (szstr) {
                   LLint size = 0;
@@ -483,10 +534,31 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                     back->r.totalsize = size;
                   }
                 }
+
+                // MDTM?
+                ftp_command(line, sizeof(line), "MDTM", ua);
+                strcpybuff(back->info, "mdtm");
+                send_line(soc_ctl, line);
+                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+                _CHECK_HALT_FTP;
+                if (ftp_parse_mdtm(line, &remote_tm)) {
+                  char date[256];
+
+                  time_rfc822(date, &remote_tm);
+                  /* Stamp the mirror as the HTTP path does, so a later pass
+                     compares server-clock times instead of crossing clocks. */
+                  back->r.lastmodified[0] = '\0';
+                  strlncatbuff(back->r.lastmodified, date,
+                               sizeof(back->r.lastmodified),
+                               sizeof(back->r.lastmodified) - 1);
+                  remote_mtime = timegm(&remote_tm);
+                }
+
                 /* Only over a copy back_add() judged partial: on --update every
                    mirrored file exists, and resuming a complete one splices the
                    old body into the new (#798). */
-                if (back->range_req_size > 0 && (transfer_list == 0)) {
+                if (back->range_req_size > 0 && (transfer_list == 0) &&
+                    ftp_may_resume(opt, back, remote_mtime)) {
                   strcpybuff(back->info, "rest");
                   snprintf(line, sizeof(line), "REST " LLintP,
                            (LLint) back->range_req_size);
@@ -695,6 +767,10 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               fclose(back->r.fp);
               back->r.fp = NULL;
             }
+            /* back_flush_output() stamps only r.is_write transfers, which FTP
+               never is; a partial needs the remote's date too (#823). */
+            if (strnotempty(back->r.lastmodified))
+              set_filetime_rfc822(back->url_sav, back->r.lastmodified);
           } else {
             strcpybuff(back->r.msg, "Unable to write file");
             back->r.statuscode = STATUSCODE_INVALID;
diff --git a/src/htslib.c b/src/htslib.c
index 3f39490b..c8608ea9 100644
--- a/src/htslib.c
+++ b/src/htslib.c
@@ -2848,6 +2848,15 @@ int set_filetime_rfc822(const char *file, const char *date) {
     return -1;
 }
 
+/* Note: utf-8 */
+time_t get_filetime(const char *file) {
+  STRUCT_STAT buf;
+
+  if (STAT(file, &buf) != 0)
+    return (time_t) -1;
+  return (time_t) buf.st_mtime;
+}
+
 /* Note: utf-8 */
 int get_filetime_rfc822(const char *file, char *date) {
   STRUCT_STAT buf;
diff --git a/src/htslib.h b/src/htslib.h
index 9ac442cb..62d00d16 100644
--- a/src/htslib.h
+++ b/src/htslib.h
@@ -278,6 +278,8 @@ void hts_now_iso8601(char out[32]);
 struct tm *convert_time_rfc822(struct tm *buffer, const char *s);
 int set_filetime(const char *file, struct tm *tm_time);
 int set_filetime_rfc822(const char *file, const char *date);
+/* File mtime as a UTC time_t, or (time_t) -1 if it can not be read. */
+time_t get_filetime(const char *file);
 int get_filetime_rfc822(const char *file, char *date);
 HTS_INLINE void time_rfc822(char *s, struct tm *A);
 HTS_INLINE void time_rfc822_local(char *s, struct tm *A);


Test patch:
diff --git a/tests/121_local-ftp-nocache-splice.test b/tests/121_local-ftp-nocache-splice.test
new file mode 100755
index 00000000..3e30dd1b
--- /dev/null
+++ b/tests/121_local-ftp-nocache-splice.test
@@ -0,0 +1,188 @@
+#!/bin/bash
+#
+# With no cache entry an FTP transfer resumed on the strength of the local
+# file's mere existence, so a changed remote got its tail spliced onto the old
+# head (#823). Resuming now needs SIZE and MDTM to agree that the remote can
+# still extend the local copy; pass 2 proves a genuine partial still resumes.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+root="${tmpdir}/root"
+out="${tmpdir}/crawl"
+mode="${tmpdir}/mode"
+cmds="${tmpdir}/cmds"
+mkdir -p "$root" "$out"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+size_of() { wc -c <"$1" | tr -d '[:space:]'; }
+# hts-log.txt is rewritten per pass, so this only ever sees the pass just run.
+no_errors() {
+    local errors
+    errors=$(grep -a 'Error:' "${out}/hts-log.txt" || true)
+    test -z "$errors" || fail "$1 reported errors: ${errors}"
+}
+# Commands the server saw since the last pass; each pass starts by emptying it.
+sent() { cat "$cmds"; }
+
+# Generations differ in their first bytes, so a splice shows wherever it lands;
+# the mtime is explicit because the fix decides on it, and a race would not do.
+write_body() {
+    "$python" -c 'import os, sys
+path, tag, fill, stamp = sys.argv[1:5]
+with open(path, "wb") as fp:
+    fp.write(("%s-BODY " % tag).encode() + tag[:1].encode() * int(fill))
+os.utime(path, (int(stamp), int(stamp)))' "${root}/a.bin" "$1" "$2" "$3"
+}
+# A copy this engine did not stamp: an archive restore, a cp without -p, or any
+# mirror predating the fix. Content is stale, so a splice fails cmp.
+plant_stale() {
+    "$python" -c 'import os, sys
+path, fill, stamp = sys.argv[1:4]
+with open(path, "wb") as fp:
+    fp.write(b"STALE-BODY " + b"S" * int(fill))
+os.utime(path, (int(stamp), int(stamp)))' "$mirror" "$1" "$2"
+}
+old_stamp=1767225600    # 2026-01-01Z
+new_stamp=1769904000    # 2026-02-01Z
+future_stamp=1772582400 # 2026-03-04Z, later than any remote below
+
+write_body OLD 8000 "$old_stamp"
+echo '/a.bin truncate' >"$mode"
+
+serverlog="${tmpdir}/server.out"
+"$python" "$server" --root "$(nativepath "$root")" \
+    --mode-file "$(nativepath "$mode")" --log "$(nativepath "$cmds")" \
+    >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 300); do
+    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "ftp server exited early: $(cat "$serverlog")" >&2
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
+    exit 1
+}
+host="127.0.0.1_${port}"
+url="ftp://127.0.0.1:${port}/a.bin"
+mirror="${out}/${host}/a.bin"
+# --cache=0 is the whole point: no cache entry to tell a partial from a stale
+# copy. Bounded like every crawl pass, so a wedge fails instead of hanging.
+crawl() { run_with_timeout 300 httrack "$url" -O "$out" --quiet \
+    --disable-security-limits --robots=0 --timeout=20 --max-time=120 --retries=1 \
+    --cache=0; }
+
+# --- pass 1: the transfer dies mid-body and leaves a partial -----------------
+crawl >"${tmpdir}/log1" 2>&1 || true
+test -f "$mirror" || fail "pass 1 mirrored nothing to resume"
+partial=$(size_of "$mirror")
+whole=$(size_of "${root}/a.bin")
+if test "$partial" -le 0 || test "$partial" -ge "$whole"; then
+    fail "pass 1 left ${partial} bytes, wanted a partial of ${whole}"
+fi
+ok "pass 1 left a ${partial}-byte partial copy"
+
+# --- pass 2: an untouched remote is still resumed ----------------------------
+: >"$mode"
+: >"$cmds"
+crawl >"${tmpdir}/log2" 2>&1
+case "$(sent)" in
+*"REST ${partial}"*) ok "pass 2 resumed at the partial's ${partial} bytes" ;;
+*) fail "pass 2 sent no REST ${partial}; commands were: $(sent)" ;;
+esac
+cmp -s "$mirror" "${root}/a.bin" ||
+    fail "the resumed copy is $(size_of "$mirror") bytes, served $(size_of "${root}/a.bin")"
+no_errors "pass 2"
+ok "the resumed transfer produced the served body and no error"
+
+# --- pass 3: a longer, newer remote over that complete copy ------------------
+write_body NEW 12000 "$new_stamp"
+: >"$cmds"
+crawl >"${tmpdir}/log3" 2>&1
+case "$(sent)" in
+*REST*) fail "resumed over a remote that had changed: $(sent)" ;;
+*RETR*) ok "the changed remote was re-fetched whole" ;;
+*) fail "pass 3 never reached RETR; commands were: $(sent)" ;;
+esac
+cmp -s "$mirror" "${root}/a.bin" ||
+    fail "the mirror is $(size_of "$mirror") bytes, served $(size_of "${root}/a.bin")"
+no_errors "pass 3"
+ok "the mirror holds the new body, not a splice of both"
+
+# --- pass 4: a remote that changed without changing length or mtime ----------
+# MDTM cannot see this one: only the size test refuses it.
+write_body ALT 12000 "$new_stamp"
+: >"$cmds"
+crawl >"${tmpdir}/log4" 2>&1
+case "$(sent)" in
+*REST*) fail "resumed over a copy the remote can no longer extend: $(sent)" ;;
+*RETR*) ok "the same-length remote was re-fetched whole" ;;
+*) fail "pass 4 never reached RETR; commands were: $(sent)" ;;
+esac
+cmp -s "$mirror" "${root}/a.bin" ||
+    fail "the mirror is $(size_of "$mirror") bytes, served $(size_of "${root}/a.bin")"
+no_errors "pass 4"
+ok "the same-length change replaced the mirror"
+
+# --- pass 5: a server with no MDTM cannot prove anything, so no resume -------
+# The copy is dated exactly as the remote, so only the missing MDTM refuses it.
+write_body NEW 12000 "$new_stamp"
+plant_stale 4000 "$new_stamp"
+echo '/a.bin nomdtm' >"$mode"
+: >"$cmds"
+crawl >"${tmpdir}/log5" 2>&1
+case "$(sent)" in
+*REST*) fail "resumed against a server that answers no MDTM: $(sent)" ;;
+*RETR*) ok "no MDTM, no resume" ;;
+*) fail "pass 5 never reached RETR; commands were: $(sent)" ;;
+esac
+cmp -s "$mirror" "${root}/a.bin" ||
+    fail "the mirror is $(size_of "$mirror") bytes, served $(size_of "${root}/a.bin")"
+no_errors "pass 5"
+ok "the unprovable resume was re-fetched whole"
+
+# --- pass 6: a copy this engine never stamped, so dated by the local clock ---
+# Every mirror made before the fix looks like this. "Not newer than the remote"
+# waves it through, which is why the date has to match rather than merely lead.
+: >"$mode"
+plant_stale 4000 "$future_stamp"
+: >"$cmds"
+crawl >"${tmpdir}/log6" 2>&1
+case "$(sent)" in
+*REST*) fail "resumed over a copy dated by the local clock: $(sent)" ;;
+*RETR*) ok "the unstamped copy was re-fetched whole" ;;
+*) fail "pass 6 never reached RETR; commands were: $(sent)" ;;
+esac
+cmp -s "$mirror" "${root}/a.bin" ||
+    fail "the mirror is $(size_of "$mirror") bytes, served $(size_of "${root}/a.bin")"
+no_errors "pass 6"
+ok "a newer local mtime does not license a resume"
diff --git a/tests/ftp-server.py b/tests/ftp-server.py
index 9c1d0fff..9cb10176 100644
--- a/tests/ftp-server.py
+++ b/tests/ftp-server.py
@@ -9,6 +9,7 @@
   truncate  send a prefix of the body, then drop the data connection
   empty     open the data connection and send nothing
   norest    answer REST with 500, so the client re-fetches from scratch
+  nomdtm    answer MDTM with 500, like a server predating RFC 3659
 """
 
 import argparse
@@ -16,6 +17,7 @@
 import socket
 import sys
 import threading
+import time
 
 
 def reply(conn, text):
@@ -148,6 +150,15 @@ def dispatch(self, verb, arg):
                 reply(conn, "213 %d" % os.path.getsize(path))
             else:
                 reply(conn, "550 no such file")
+        elif verb == "MDTM":
+            path = self.resolve(arg)
+            if "nomdtm" in self.modes():
+                reply(conn, "500 MDTM not understood")
+            elif path and os.path.isfile(path):
+                stamp = time.gmtime(os.path.getmtime(path))
+                reply(conn, "213 " + time.strftime("%Y%m%d%H%M%S", stamp))
+            else:
+                reply(conn, "550 no such file")
         elif verb == "REST":
             if "norest" in self.modes():
                 reply(conn, "500 REST not understood")
diff --git a/tests/tests-list.mk b/tests/tests-list.mk
index e4d6d2cb..5e285b9a 100644
--- a/tests/tests-list.mk
+++ b/tests/tests-list.mk
@@ -96,6 +96,7 @@ TESTS += 110_local-ftp-parallel.test
 TESTS += 111_local-ftp-update-rest.test
 TESTS += 114_local-update-304-leak.test
 TESTS += 117_local-proxytrack-ndx-cursor.test
+TESTS += 121_local-ftp-nocache-splice.test
 TESTS += 11_crawl-cookies.test
 TESTS += 11_crawl-idna.test
 TESTS += 11_crawl-international.test