Repository: xroche/httrack (2026-07-11T14:53:13Z)

Issue:
-M overall-size cap measures saved bytes, undercounting redirect/plugin-heavy crawls
The `-M` / `--max-size` cap is enforced against `HTS_STAT.stat_bytes`, which only counts bytes written to disk for HTTP 200 responses (minus robots.txt). On crawls dominated by redirects, error bodies, or plugin-written files, that counter lags well behind the volume actually pulled off the network (`HTS_STAT.HTS_TOTAL_RECV`). In #77 the reporter saw about 12.5 MB on disk while "Bytes saved" hovered near the 3 MB cap, so the cap barely bit.

#519 fixed the enforcement half of #77: the cap now hard-aborts the in-flight transfers instead of letting the backlog drain. It left the meter alone on purpose, since changing what a documented option measures is a behavior change worth its own review.

Choices to weigh:

- Enforce against `HTS_TOTAL_RECV` (the real received volume, monotonic, matches the "overall size that can be uploaded/scanned" wording).
- Keep `stat_bytes` but also count saved non-200 bodies.
- Cap on both, whichever trips first.

Related display point: the "Bytes saved" figure shown during the crawl is `stat_bytes` plus the sizes of in-flight transfers, so it dips whenever an in-flight transfer resolves as a redirect or error. That is the up-and-down the reporter noticed. Worth deciding whether to display the committed figure instead.


Discussion:

Patch:
diff --git a/src/htsback.c b/src/htsback.c
index 8436d73a8..c883ee4c6 100644
--- a/src/htsback.c
+++ b/src/htsback.c
@@ -4178,10 +4178,13 @@ static int back_maxtime_grace(const int maxtime) {
    No floor (unlike the time grace): a size overrun should abort promptly. */
 static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
 
-/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481). */
+/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481).
+   -M measures received volume (HTS_TOTAL_RECV), not saved 200-only stat_bytes
+   which undercounts redirect/error-heavy crawls (#520). */
 static hts_mirror_limit back_mirror_limit(httrackp *opt) {
-  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
-      HTS_STAT.stat_bytes - opt->maxsite >= back_maxsize_grace(opt->maxsite))
+  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
+      HTS_STAT.HTS_TOTAL_RECV - opt->maxsite >=
+          back_maxsize_grace(opt->maxsite))
     return HTS_MIRROR_LIMIT_SIZE;
   if (opt->maxtime > 0) {
     const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
@@ -4195,7 +4198,7 @@ static hts_mirror_limit back_mirror_limit(httrackp *opt) {
 
 int back_checkmirror(httrackp *opt) {
   /* request a smooth stop the first time each cap is reached */
-  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
+  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
       !opt->state.stop) {
     hts_log_print(opt, LOG_ERROR,
                   "More than " LLintP


Test patch:
diff --git a/tests/45_local-maxsize-recv.test b/tests/45_local-maxsize-recv.test
new file mode 100755
index 000000000..2ce04bcef
--- /dev/null
+++ b/tests/45_local-maxsize-recv.test
@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# -M caps received volume, not saved 200-only bytes (#520): links to large 404
+# bodies pump received >> saved; the cap must trip though little lands on disk.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# The mirror bound is the load-bearing half: it fires while <100 KB is saved,
+# so the cap can only have tripped on received volume, not saved bytes (which
+# would exceed the cap and fetch all 16 links).
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-found 'More than 500000 bytes have been transferred.. giving up' \
+    --max-mirror-bytes 100000 \
+    httrack 'BASEURL/maxrecv/index.html' -M500000 -c4
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 5ff0c2cf9..32e10df58 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -113,6 +113,7 @@ TESTS = \
 	41_local-utf8-link.test \
 	42_local-maxsize-slow.test \
 	43_local-update-truncate.test \
-	44_local-update-errormask.test
+	44_local-update-errormask.test \
+	45_local-maxsize-recv.test
 
 CLEANFILES = check-network_sh.cache
diff --git a/tests/local-server.py b/tests/local-server.py
index 9e1e5737c..85f908286 100755
--- a/tests/local-server.py
+++ b/tests/local-server.py
@@ -1104,6 +1104,22 @@ def route_bigtrunc_slow(self):
         except OSError:
             pass
 
+    # -M received-volume cap (#520): links to large 404 bodies. httrack receives
+    # each (HTS_TOTAL_RECV climbs) but saves none, so saved stays far below -M.
+    def route_maxrecv_index(self):
+        self.send_html(
+            "".join('\t<a href="r%d.bin">r%d</a>\n' % (i, i) for i in range(16))
+        )
+
+    def route_maxrecv_404(self):
+        body = b"x" * self.BIGFILE_BYTES
+        self.send_response(404, "Not Found")
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(body)
+
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -1206,6 +1222,23 @@ def route_bigtrunc_slow(self):
         "/mini304/page.html": route_mini304_page,
         "/errmask/index.html": route_errmask_index,
         "/errmask/keep.dat": route_errmask_keep,
+        "/maxrecv/index.html": route_maxrecv_index,
+        "/maxrecv/r0.bin": route_maxrecv_404,
+        "/maxrecv/r1.bin": route_maxrecv_404,
+        "/maxrecv/r2.bin": route_maxrecv_404,
+        "/maxrecv/r3.bin": route_maxrecv_404,
+        "/maxrecv/r4.bin": route_maxrecv_404,
+        "/maxrecv/r5.bin": route_maxrecv_404,
+        "/maxrecv/r6.bin": route_maxrecv_404,
+        "/maxrecv/r7.bin": route_maxrecv_404,
+        "/maxrecv/r8.bin": route_maxrecv_404,
+        "/maxrecv/r9.bin": route_maxrecv_404,
+        "/maxrecv/r10.bin": route_maxrecv_404,
+        "/maxrecv/r11.bin": route_maxrecv_404,
+        "/maxrecv/r12.bin": route_maxrecv_404,
+        "/maxrecv/r13.bin": route_maxrecv_404,
+        "/maxrecv/r14.bin": route_maxrecv_404,
+        "/maxrecv/r15.bin": route_maxrecv_404,
     }
 
     # --- /big/ seeded pseudo-site ------------------------------------------