Repository: xroche/httrack (2026-07-27T08:33:52Z)

Issue:
server-not-modified revisit records carry no WARC-Refers-To-Target-URI, so replay cannot resolve them
A 304 revisit is written with a `WARC-Profile` and nothing else that says what it stands for:

```
WARC-Type: revisit
WARC-Target-URI: http://host/mini304/index.html
WARC-Profile: http://netpreserve.org/warc/1.1/revisit/server-not-modified
WARC-Block-Digest: sha1:F4CY...

HTTP/1.1 200 OK
Content-Type: text/html
```

No `WARC-Refers-To`, no `WARC-Refers-To-Target-URI`, no `WARC-Refers-To-Date`, no `WARC-Payload-Digest`. In `warc_write_transaction()` (`src/htswarc.c`) the `is_update_unchanged` branch sets `profile` and leaves `refers_uri`/`refers_date` NULL, and `have_pdig` is 0 because a 304 carries no payload to digest. The identical-payload-digest branch just below fills both in from `w->seen`.

WARC 1.1 6.7.3 only recommends the field, so the record is conformant. It is still unresolvable in practice: pywb reads `WARC-Refers-To-Target-URI` in `resolvingloader.py`, wabac.js reads `record.warcRefersToTargetURI` in `warcloader.ts`, and neither looks at `WARC-Profile` or `WARC-Refers-To`. Without the target URI, pywb raises `ArchiveLoadFailed("Original for revisit record could not be loaded")` and answers 503, and wabac.js falls through to `notFoundByTypeResponse` and shows "Archived Page Not Found". Our CDXJ line for the record carries `"status": "200"` and no `digest`, so pywb's digest-keyed lookup cannot recover it either.

Both values are on hand where the record is written: the referred-to URI is the record's own target URI, and the date is the cached entry's.

#759 is why this is worth fixing now. Whichever way that lands, a second pass whose revisits nobody can resolve has little value, so this wants doing before either the append or the segment-per-run option is built.



Discussion:

Patch:
diff --git a/src/htswarc.c b/src/htswarc.c
index 4cf3efe7..c3cb1c45 100644
--- a/src/htswarc.c
+++ b/src/htswarc.c
@@ -127,24 +127,32 @@ static void wbuf_free(wbuf *b) {
   b->len = b->cap = 0;
 }
 
-/* Append n bytes; returns 0 on success, -1 on OOM/overflow. */
-static int wbuf_add(wbuf *b, const void *p, size_t n) {
+/* Make room for n more bytes; returns 0 on success, -1 on OOM/overflow. */
+static int wbuf_reserve(wbuf *b, size_t n) {
+  size_t ncap;
+  char *nd;
   if (n > (size_t) -1 - b->len)
     return -1;
-  if (b->len + n > b->cap) {
-    size_t ncap = b->cap ? b->cap : 256;
-    char *nd;
-    while (ncap < b->len + n) {
-      if (ncap > (size_t) -1 / 2)
-        return -1;
-      ncap *= 2;
-    }
-    nd = realloct(b->data, ncap);
-    if (nd == NULL)
+  if (b->len + n <= b->cap)
+    return 0;
+  ncap = b->cap ? b->cap : 256;
+  while (ncap < b->len + n) {
+    if (ncap > (size_t) -1 / 2)
       return -1;
-    b->data = nd;
-    b->cap = ncap;
+    ncap *= 2;
   }
+  nd = realloct(b->data, ncap);
+  if (nd == NULL)
+    return -1;
+  b->data = nd;
+  b->cap = ncap;
+  return 0;
+}
+
+/* Append n bytes; returns 0 on success, -1 on OOM/overflow. */
+static int wbuf_add(wbuf *b, const void *p, size_t n) {
+  if (wbuf_reserve(b, n) != 0)
+    return -1;
   memcpy(b->data + b->len, p, n);
   b->len += n;
   return 0;
@@ -156,16 +164,32 @@ static int wbuf_puts(wbuf *b, const char *s) {
 
 static int wbuf_printf(wbuf *b, const char *fmt, ...) HTS_PRINTF_FUN(2, 3);
 
+/* A header line carrying a URL has no useful bound, and giving up here loses
+   the whole record, so anything past the stack buffer is formatted into the
+   wbuf itself rather than rejected (#785). */
 static int wbuf_printf(wbuf *b, const char *fmt, ...) {
   char tmp[1024];
+  size_t need;
   int n;
   va_list ap;
   va_start(ap, fmt);
   n = vsnprintf(tmp, sizeof(tmp), fmt, ap);
   va_end(ap);
-  if (n < 0 || (size_t) n >= sizeof(tmp))
+  if (n < 0)
+    return -1;
+  if ((size_t) n < sizeof(tmp))
+    return wbuf_add(b, tmp, (size_t) n);
+  need = (size_t) n + 1; /* +1: vsnprintf always writes the NUL */
+  if (wbuf_reserve(b, need) != 0)
+    return -1;
+  va_start(ap, fmt);
+  n = vsnprintf(b->data + b->len, need, fmt, ap);
+  va_end(ap);
+  /* Never advance past what was reserved, whatever the second pass returns. */
+  if (n < 0 || (size_t) n >= need)
     return -1;
-  return wbuf_add(b, tmp, (size_t) n);
+  b->len += (size_t) n; /* the NUL is scratch, overwritten by the next append */
+  return 0;
 }
 
 /* ---- gzip-per-record member writer (mirrors ae_write_packed) ---- */
@@ -1684,6 +1708,10 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
   if (is_update_unchanged) {
     is_revisit = 1;
     profile = "http://netpreserve.org/warc/1.1/revisit/server-not-modified";
+    /* Replay resolves a revisit by this field alone, and a 304 stands in for
+       the same URL. No WARC-Refers-To-Date to go with it: the cache keeps the
+       document's Last-Modified, never the previous capture time. */
+    refers_uri = target_uri;
     /* Served from cache: the payload sits in the previous archive, not here. */
     w->unbacked_revisits++;
   } else if (have_pdig && w->seen != NULL) {


Test patch:
diff --git a/src/htsselftest.c b/src/htsselftest.c
index 1c8eb9c0..f7fa4381 100644
--- a/src/htsselftest.c
+++ b/src/htsselftest.c
@@ -4944,6 +4944,101 @@ static int st_warc_surt(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+/* A URL longer than the old 1024-byte header-format buffer must still reach the
+   archive: the record used to be abandoned whole, silently (#785). The sweep
+   straddles the boundary so both the stack-buffer and the grow path run. */
+static int st_warc_longurl(httrackp *opt, int argc, char **argv) {
+  /* "WARC-Target-URI: " + CRLF costs 19 bytes, so the old buffer failed at
+     1005; 9000 forces several reallocs within one record. */
+  static const size_t lengths[] = {100, 1003, 1004, 1005, 1006, 2000, 9000};
+  static const char resp_hdr[] =
+      "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n";
+  char path[HTS_URLMAXSIZE * 2];
+  char body[64];
+  warc_writer *w;
+  FILE *fp;
+  char *blob;
+  LLint fsz;
+  const char *at2;
+  size_t i, n, nrec = 0;
+  int err = 0;
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-longurl: need a writable directory\n");
+    return 1;
+  }
+  snprintf(path, sizeof(path), "%s/longurl.warc", argv[0]);
+
+  w = warc_open(opt, path);
+  if (w == NULL) {
+    fprintf(stderr, "warc-longurl: could not create %s\n", path);
+    return 1;
+  }
+  for (i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) {
+    const size_t len = lengths[i];
+    char *uri = malloct(len + 1);
+
+    if (uri == NULL) {
+      warc_close(w);
+      return 1;
+    }
+    /* A distinct tail per URI so a truncated one cannot match another. */
+    snprintf(uri, len + 1, "http://example.com/%04d/", (int) len);
+    memset(uri + strlen(uri), 'a', len - strlen(uri));
+    uri[len] = '\0';
+    /* Distinct payloads: identical ones dedupe into revisit records. */
+    snprintf(body, sizeof(body), "<html><body>%04d</body></html>\n", (int) len);
+    if (warc_write_transaction(w, uri, NULL, NULL, resp_hdr, body, strlen(body),
+                               NULL, 200, 0, 0) != 0) {
+      fprintf(stderr, "warc-longurl: write failed at length %d\n", (int) len);
+      err = 1;
+    }
+    freet(uri);
+  }
+  warc_close(w);
+
+  fsz = fsize_utf8(path);
+  blob = (fsz > 0) ? malloct((size_t) fsz + 1) : NULL;
+  if (blob == NULL) {
+    fprintf(stderr, "warc-longurl: no archive written\n");
+    return 1;
+  }
+  fp = FOPEN(path, "rb");
+  n = (fp != NULL) ? fread(blob, 1, (size_t) fsz, fp) : 0;
+  if (fp != NULL)
+    fclose(fp);
+  blob[n] = '\0';
+
+  for (i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) {
+    const size_t len = lengths[i];
+    char want[64];
+    const char *at;
+
+    snprintf(want, sizeof(want), "WARC-Target-URI: http://example.com/%04d/",
+             (int) len);
+    at = strstr(blob, want);
+    if (at == NULL) {
+      fprintf(stderr, "warc-longurl: length %d lost its record\n", (int) len);
+      err = 1;
+    } else if (strlen(at) < strlen("WARC-Target-URI: ") + len ||
+               at[strlen("WARC-Target-URI: ") + len] != '\r') {
+      fprintf(stderr, "warc-longurl: length %d truncated\n", (int) len);
+      err = 1;
+    }
+  }
+  for (at2 = blob; (at2 = strstr(at2, "WARC-Type: response")) != NULL; at2++)
+    nrec++;
+  if (nrec != sizeof(lengths) / sizeof(lengths[0])) {
+    fprintf(stderr, "warc-longurl: %d response records, want %d\n", (int) nrec,
+            (int) (sizeof(lengths) / sizeof(lengths[0])));
+    err = 1;
+  }
+
+  freet(blob);
+  printf("warc-longurl: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 /* End-to-end CDXJ: crawl a handful of records with --warc-cdx, then verify the
    .cdx is sorted, has exactly one line per response/revisit/resource (none for
    warcinfo/request), and each offset/length points at a gzip member that
@@ -6787,6 +6882,9 @@ static const struct selftest_entry {
      st_warc_verbatim},
     {"warc-surt", "", "SURT canonicalization of the CDXJ sort key",
      st_warc_surt},
+    {"warc-longurl", "<dir>",
+     "a URL past the header-format buffer still reaches the archive",
+     st_warc_longurl},
     {"longpath", "<dir>",
      "round-trip a >MAX_PATH file through the _w* wrappers (\\\\?\\ on "
      "Windows)",
diff --git a/tests/104_engine-warc-longurl.test b/tests/104_engine-warc-longurl.test
new file mode 100644
index 00000000..3d629bfb
--- /dev/null
+++ b/tests/104_engine-warc-longurl.test
@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# A URL past the header-format buffer must still produce a record: the archive
+# used to lose it whole, with no error and an exit status of 0 (#785).
+
+set -e
+
+: "${top_srcdir:=..}"
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_longurl.XXXXXX")
+trap 'rm -rf "$tmp"' EXIT
+
+httrack -O "$tmp/out" "-#test=warc-longurl" "$tmp"
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 227452b0..3465a095 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -217,6 +217,7 @@ TESTS = \
 	99_local-robots-error.test \
 	100_local-purge-longpath.test \
 	101_local-update-stale-bak.test \
-	102_local-ftp-refetch.test
+	102_local-ftp-refetch.test \
+	104_engine-warc-longurl.test
 
 CLEANFILES = check-network_sh.cache
diff --git a/tests/warc-validate.py b/tests/warc-validate.py
index da476714..e10540a9 100755
--- a/tests/warc-validate.py
+++ b/tests/warc-validate.py
@@ -160,6 +160,19 @@ def main():
                     body_hits[sub] = True
         elif wtype == b"revisit":
             revisits += 1
+            # WARC 1.1 5.12: a revisit names the capture it stands for. Without it
+            # the record is well-formed and unreplayable, since pywb raises
+            # ArchiveLoadFailed and wabac.js answers Not Found.
+            refers = field(header, b"WARC-Refers-To-Target-URI")
+            shown = uri.decode("utf-8", "replace")
+            if not refers:
+                sys.exit("revisit for %s has no WARC-Refers-To-Target-URI" % shown)
+            profile = field(header, b"WARC-Profile") or b""
+            if b"server-not-modified" in profile and refers != uri:
+                sys.exit(
+                    "server-not-modified revisit for %s refers to %s"
+                    % (shown, refers.decode("utf-8", "replace"))
+                )
             for sub in no_resp:
                 if sub.encode() in uri:
                     revisit_hits[sub] = True