Repository: xroche/httrack (2026-07-27T19:21:45Z)

Issue:
A 304 revisit record names no peer IP
`warc_write_backtransaction()` reads the peer address from `back->r.address`, which the 304 path in `back_wait()` loses along with everything else the cache entry overwrites. `back_connxfr()` moves the socket and keep-alive members across the swap, not the address, so every revisit record comes out without the `WARC-IP-Address` a full response carries.

Visible on the mini304 fixture: crawl it with `--warc-file`, re-crawl with `--update --warc-file <other name>`, and the revisit records in the second archive have no `WARC-IP-Address` line while the first archive's response records do.

Found while fixing #826, which moves the stashed request and response headers across the same swap.
An engine-forced not-modified is archived as a server-not-modified revisit the server never sent
`back_wait()` forces `HTTP_NOT_MODIFIED` on its own in several places where the server said something else: the same-size hack, and the #176 path that ignores an error on update to keep the existing copy. The 304 branch then sets `notmodified`, and `warc_write_backtransaction()` turns that into a `revisit` record whose `WARC-Profile` is `.../revisit/server-not-modified`.

Nothing behind it is a 304. The block is a synthesized `HTTP/1.1 200 OK`, and the real exchange was a 200 with a body or a 403. A replay engine resolving that profile expects a server that answered 304, and `tests/warc-validate.py --revisit-exchange` (added in #837) rejects such a record.

Either the record should be an `identical-payload-digest` revisit against the previous capture, or the forced case should write no record at all and let the previous archive stand. Found while fixing #826; the fix there gates the header carry-over on `server_sent_304` so this case keeps its current output rather than gaining a contradictory one.


Discussion:

Patch:
diff --git a/src/htsback.c b/src/htsback.c
index 2eccedc6..a59072a0 100644
--- a/src/htsback.c
+++ b/src/htsback.c
@@ -1283,6 +1283,7 @@ void back_connxfr(htsblk * src, htsblk * dst) {
   src->keep_alive_t = 0;
   dst->debugid = src->debugid;
   src->debugid = 0;
+  dst->address = src->address; // peer IP survives the cache-entry swap (#838)
 }
 
 /* Release the buffers a response owns. The connection members are left alone:
@@ -2138,6 +2139,8 @@ int back_add(struct_back *sback, httrackp *opt, cache_back *cache,
                             back[p].url_adr, back[p].url_fil);
             }
             back[p].r.notmodified = 1;  // fichier non modifié
+            // no request was sent at all, so this is never a server 304 (#839)
+            back[p].r.warc_forced_notmodified = HTS_TRUE;
             back[p].status = STATUS_READY; // OK prêt
             back_set_finished(sback, p);
 
@@ -4184,6 +4187,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       back[i].status = STATUS_READY;    // OK prêt
                       back_set_finished(sback, i);
                       back[i].r.notmodified = 1;        // NON modifié!
+                      // WARC must not claim a 304 the server never sent (#839)
+                      back[i].r.warc_forced_notmodified =
+                          server_sent_304 ? HTS_FALSE : HTS_TRUE;
                       hts_log_print(opt, LOG_DEBUG,
                                     "File loaded after test from cache: %s%s",
                                     back[i].url_adr, back[i].url_fil);
diff --git a/src/htsopt.h b/src/htsopt.h
index 365a77be..a4d96530 100644
--- a/src/htsopt.h
+++ b/src/htsopt.h
@@ -695,6 +695,9 @@ struct htsblk {
   char *warc_rawpath; /**< verbatim WARC: spooled compressed body path, or NULL
                            (owns the file; unlinked on free). Tail: ABI */
   LLint warc_rawsize; /**< byte length of warc_rawpath. Tail: ABI */
+  /** notmodified came from an engine hack, not a server 304 (#839).
+      Tail: ABI */
+  hts_boolean warc_forced_notmodified;
   /*char digest[32+2];   // md5 digest generated by the engine ("" if none) */
 };
 
diff --git a/src/htswarc.c b/src/htswarc.c
index 3b02f5b5..8a1a5865 100644
--- a/src/htswarc.c
+++ b/src/htswarc.c
@@ -94,7 +94,8 @@ struct warc_writer {
   hts_boolean protect_prev;   /* previous archive present: build in a temp */
   hts_boolean opened;         /* open completed; a failed one swaps nothing */
   hts_boolean failed;         /* a record or segment was lost: swap nothing */
-  uint64_t unbacked_revisits; /* revisits whose payload no file here holds */
+  uint64_t unbacked_revisits; /* URLs this pass didn't capture here: an
+                                  unbacked revisit, or nothing written at all */
   char **page_lines; /* one JSON page line per 200 text/html response, owned */
   size_t page_count;
   size_t page_cap;
@@ -1612,7 +1613,7 @@ static hts_boolean warc_commit(warc_writer *w) {
     hts_log_print(
         w->opt, LOG_ERROR,
         "WARC: this pass revisited %llu URL(s) without re-downloading them, so "
-        "its archive would name bodies no file holds; kept the previous %s "
+        "this archive doesn't hold their current content; kept the previous %s "
         "(re-run with -C0, or --warc-file with a name of its own)",
         (unsigned long long) w->unbacked_revisits,
         warc_seg_path(w, 0, finalbuf, sizeof(finalbuf)));
@@ -1678,7 +1679,7 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
                            const char *resp_hdr, const char *body,
                            size_t body_len, const char *body_path,
                            const char *content_type, int statuscode,
-                           int is_update_unchanged, int truncated) {
+                           int unchanged_kind, int truncated) {
   wbuf http;
   char resp_id[64];
   char pdig[33];
@@ -1707,13 +1708,13 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
 
   /* A payload exists (for digesting) unless this is a bodyless 304. */
   has_payload = (body_len > 0 && (body != NULL || body_path != NULL) &&
-                 !is_update_unchanged);
+                 unchanged_kind != WARC_UNCHANGED_SERVER_304);
 
   /* Payload digest drives identical-payload-digest dedup (OpenSSL only). */
   have_pdig =
       has_payload ? payload_digest_b32(body, body_len, body_path, pdig) : 0;
 
-  if (is_update_unchanged) {
+  if (unchanged_kind == WARC_UNCHANGED_SERVER_304) {
     is_revisit = 1;
     profile = "http://netpreserve.org/warc/1.1/revisit/server-not-modified";
     /* Replay resolves a revisit by this field alone, and a 304 stands in for
@@ -1722,6 +1723,22 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
     refers_uri = target_uri;
     /* Served from cache: the payload sits in the previous archive, not here. */
     w->unbacked_revisits++;
+  } else if (unchanged_kind == WARC_UNCHANGED_ENGINE_FORCED) {
+    /* has_payload requires body_len>0, so a genuinely empty body looks
+       digest-less too; it still has a well-defined digest (sha1 of nothing),
+       so compute it here rather than treat it as a missing-crypto case. */
+    if (!have_pdig && body_len == 0 && (body != NULL || body_path != NULL))
+      have_pdig = payload_digest_b32(body, body_len, body_path, pdig);
+    /* Served from cache with no exchange either way: still unbacked. */
+    w->unbacked_revisits++;
+    /* No digest (no OpenSSL) means nothing to point a revisit at, and there
+       was no real exchange to record as a response; write nothing (#839). */
+    if (!have_pdig)
+      return 0;
+    is_revisit = 1;
+    profile =
+        "http://netpreserve.org/warc/1.1/revisit/identical-payload-digest";
+    refers_uri = target_uri;
   } else if (have_pdig && w->seen != NULL) {
     void *prev = NULL;
     if (coucal_read_pvoid(w->seen, pdig, &prev) && prev != NULL) {
@@ -1831,7 +1848,7 @@ void warc_write_backtransaction(httrackp *opt, lien_back *back) {
   const char *body_path;
   const char *resp_hdr;
   char synth[512];
-  int is_unchanged;
+  int unchanged_kind;
   int is_ftp;
 
   if (opt->state.warc == WARC_DISABLED)
@@ -1892,7 +1909,12 @@ void warc_write_backtransaction(httrackp *opt, lien_back *back) {
     body_len = (size_t) back->r.warc_rawsize;
   }
 
-  is_unchanged = (back->r.notmodified && opt->is_update) ? 1 : 0;
+  if (!back->r.notmodified || !opt->is_update)
+    unchanged_kind = WARC_UNCHANGED_NONE;
+  else if (back->r.warc_forced_notmodified)
+    unchanged_kind = WARC_UNCHANGED_ENGINE_FORCED;
+  else
+    unchanged_kind = WARC_UNCHANGED_SERVER_304;
 
   /* Prefer the stashed raw headers; synthesize a minimal status line for the
      header-less (HTTP/0.9-style) responses that never carried a header block.
@@ -1908,6 +1930,6 @@ void warc_write_backtransaction(httrackp *opt, lien_back *back) {
 
   warc_write_transaction(w, uri, ip, back->r.warc_reqhdr, resp_hdr, body,
                          body_len, body_path, back->r.contenttype,
-                         back->r.statuscode, is_unchanged,
+                         back->r.statuscode, unchanged_kind,
                          back->r.warc_truncated);
 }
diff --git a/src/htswarc.h b/src/htswarc.h
index b913ba80..a5fe9632 100644
--- a/src/htswarc.h
+++ b/src/htswarc.h
@@ -101,6 +101,12 @@ void warc_close(warc_writer *w);
    success, -1 on error or truncation. Exposed for the -#test=warc-surt test. */
 int warc_surt(const char *url, char *out, size_t outsz);
 
+/* unchanged_kind for warc_write_transaction. */
+#define WARC_UNCHANGED_NONE 0       /* a fresh exchange: a full response */
+#define WARC_UNCHANGED_SERVER_304 1 /* the server itself answered 304 */
+#define WARC_UNCHANGED_ENGINE_FORCED                                           \
+  2 /* the engine, not the server, decided nothing changed (#839) */
+
 /* Write one transaction's request + response (or revisit) records.
    target_uri:  absolute URL fetched.
    ip:          numeric peer IP, or NULL/"" to omit.
@@ -110,7 +116,9 @@ int warc_surt(const char *url, char *out, size_t outsz);
    body_path:   file re-read for the body when body==NULL (may be NULL).
    content_type: CDXJ mime when resp_hdr declares none, as a 304 does; NULL
                 to leave the index line without one.
-   is_update_unchanged: nonzero for a 304 server-not-modified revisit.
+   unchanged_kind: WARC_UNCHANGED_*; SERVER_304 writes a server-not-modified
+                revisit, ENGINE_FORCED a self-referencing identical-payload-
+                digest one (no 304 to claim), both unbacked (#839).
    truncated: a WARC_TRUNC_* reason to tag a cap-truncated body, else 0.
    The body is stored verbatim: Content-Encoding is kept and Content-Length set
    to body_len, so body/body_len must be the as-received (coded) bytes.
@@ -120,7 +128,7 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
                            const char *resp_hdr, const char *body,
                            size_t body_len, const char *body_path,
                            const char *content_type, int statuscode,
-                           int is_update_unchanged, int truncated);
+                           int unchanged_kind, int truncated);
 
 /* Write one non-HTTP capture as a single WARC 'resource' record: the block is
    the raw payload (no HTTP envelope), Content-Type is the payload's own MIME.


Test patch:
diff --git a/tests/134_local-warc-revisit-ip.test b/tests/134_local-warc-revisit-ip.test
new file mode 100755
index 00000000..91590d7f
--- /dev/null
+++ b/tests/134_local-warc-revisit-ip.test
@@ -0,0 +1,15 @@
+#!/bin/bash
+# A 304 revisit record must name the peer it revisited (#838).
+set -eu
+
+: "${top_srcdir:=..}"
+
+export WARC_VALIDATE_UPDATE="warc-revisit.warc.gz"
+export WARC_VALIDATE_NORESP="mini304/index.html mini304/page.html"
+export WARC_VALIDATE_IP="mini304/index.html=127.0.0.1 mini304/page.html=127.0.0.1"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --warc-validate \
+    --rerun-args '--warc-file warc-revisit' \
+    --log-found 'no files updated' \
+    --found 'mini304/index.html' --found 'mini304/page.html' \
+    httrack 'BASEURL/mini304/index.html' --warc-file warc-out
diff --git a/tests/135_local-warc-forced-notmodified.test b/tests/135_local-warc-forced-notmodified.test
new file mode 100755
index 00000000..73358a31
--- /dev/null
+++ b/tests/135_local-warc-forced-notmodified.test
@@ -0,0 +1,37 @@
+#!/bin/bash
+# An engine-forced not-modified is not a 304 the server sent (#839).
+set -eu
+
+: "${top_srcdir:=..}"
+
+# Scenario 1: real 304 (mini304) and engine-forced (errmask) side by side.
+export WARC_VALIDATE_UPDATE="warc-forced.warc.gz"
+export WARC_VALIDATE_EXCHANGE=1
+if test "${HTTPS_SUPPORT:-}" = "no"; then
+    # no OpenSSL: no payload digest to point a revisit at, so no record.
+    export WARC_VALIDATE_NORESP="mini304/index.html mini304/page.html"
+    export WARC_VALIDATE_NORECORD="errmask/keep.dat errmask/empty.dat"
+    export WARC_VALIDATE_PROFILE="mini304/index.html=server-not-modified mini304/page.html=server-not-modified"
+else
+    # empty.dat is a genuinely zero-length body: it still has a well-defined
+    # digest and must get its own revisit, not fall into the no-crypto case.
+    export WARC_VALIDATE_NORESP="mini304/index.html mini304/page.html errmask/keep.dat errmask/empty.dat"
+    export WARC_VALIDATE_PROFILE="mini304/index.html=server-not-modified mini304/page.html=server-not-modified errmask/keep.dat=identical-payload-digest errmask/empty.dat=identical-payload-digest"
+fi
+
+bash "$top_srcdir/tests/local-crawl.sh" --warc-validate \
+    --rerun-args '--warc-file warc-forced' \
+    --found 'mini304/index.html' --found 'mini304/page.html' \
+    --found 'errmask/keep.dat' --found 'errmask/empty.dat' \
+    --file-matches 'errmask/keep.dat' '^KEEP' \
+    --file-not-matches 'errmask/keep.dat' 'error 403' \
+    httrack 'BASEURL/mini304/index.html' 'BASEURL/errmask/index.html' --warc-file warc-out
+
+# Scenario 2: an all-forced pass, same archive name as pass 1, must not
+# replace the previous archive with an (almost) empty one (#777's shape).
+# No --wacz: that needs OpenSSL and would skip this scenario on a build
+# without it, hiding the one thing it exists to catch.
+bash "$top_srcdir/tests/local-crawl.sh" --archive-kept-on-rerun \
+    --archive-min-files 2 --rerun \
+    --found 'errmask/keep.dat' \
+    httrack 'BASEURL/errmask/index.html' --warc-file warc-out --warc-cdx
diff --git a/tests/136_local-warc-cache-priority.test b/tests/136_local-warc-cache-priority.test
new file mode 100755
index 00000000..abf4b7cc
--- /dev/null
+++ b/tests/136_local-warc-cache-priority.test
@@ -0,0 +1,26 @@
+#!/bin/bash
+# A cache-priority resume (-C1) never sent a request either: it must not be
+# archived as a server-not-modified revisit (#839).
+set -eu
+
+: "${top_srcdir:=..}"
+
+export WARC_VALIDATE_UPDATE="warc-cprio.warc.gz"
+export WARC_VALIDATE_EXCHANGE=1
+# Positive control on the fresh (pass-1) archive: a write-nothing regression
+# in warc_write_transaction would otherwise slip past the update-pass checks
+# below, which only assert the ABSENCE of certain records.
+export WARC_VALIDATE_BODY="mini304/index.html=3c68746d6c3e3c626f64793e0a093c6120687265663d22706167652e68746d6c223e706167653c2f613e0a3c2f626f64793e3c2f68746d6c3e0a"
+if test "${HTTPS_SUPPORT:-}" = "no"; then
+    # no OpenSSL: no payload digest to point a revisit at, so no record.
+    export WARC_VALIDATE_NORECORD="mini304/index.html mini304/page.html"
+    export WARC_VALIDATE_NO_REVISIT=1
+else
+    export WARC_VALIDATE_NORESP="mini304/index.html mini304/page.html"
+    export WARC_VALIDATE_PROFILE="mini304/index.html=identical-payload-digest mini304/page.html=identical-payload-digest"
+fi
+
+bash "$top_srcdir/tests/local-crawl.sh" --warc-validate \
+    --rerun-args '-C1 --warc-file warc-cprio' \
+    --found 'mini304/index.html' --found 'mini304/page.html' \
+    httrack 'BASEURL/mini304/index.html' --warc-file warc-out
diff --git a/tests/local-crawl.sh b/tests/local-crawl.sh
index d085bfc5..37e15f66 100755
--- a/tests/local-crawl.sh
+++ b/tests/local-crawl.sh
@@ -463,6 +463,11 @@ debug "host root: $hostroot"
 # --- optional WARC validation (stdlib validator, no warcio) ------------------
 # WARC_VALIDATE_BODY="URLSUB=HEX" byte-checks a fresh-crawl response body;
 # WARC_VALIDATE_NORESP="URLSUB..." asserts those assets are revisits post-update;
+# WARC_VALIDATE_NORECORD="URLSUB..." asserts those assets have no record at all;
+# WARC_VALIDATE_IP="URLSUB=IP..." asserts the exact WARC-IP-Address on the record;
+# WARC_VALIDATE_PROFILE="URLSUB=SUBSTR..." asserts a revisit's WARC-Profile;
+# WARC_VALIDATE_NO_REVISIT=1 skips the "at least one revisit" requirement (a
+# no-OpenSSL leg where the only unchanged assets end up with no record at all);
 # WARC_VALIDATE_EXCHANGE=1 asserts each revisit carries its 304 request/response.
 if test -n "$warc_validate"; then
     validator=$(nativepath "${testdir}/warc-validate.py")
@@ -494,10 +499,20 @@ if test -n "$warc_validate"; then
     if test -n "${WARC_VALIDATE_UPDATE:-}"; then
         upd=$(find "$mirrorroot" -maxdepth 2 -name "$WARC_VALIDATE_UPDATE" 2>/dev/null | head -n1)
         test -n "$upd" || die "no $WARC_VALIDATE_UPDATE produced under $mirrorroot"
-        declare -a revargs=(--expect-revisit)
+        declare -a revargs=()
+        test -z "${WARC_VALIDATE_NO_REVISIT:-}" && revargs+=(--expect-revisit)
         for sub in ${WARC_VALIDATE_NORESP:-}; do
             revargs+=(--no-response-for "$sub")
         done
+        for sub in ${WARC_VALIDATE_NORECORD:-}; do
+            revargs+=(--no-record-for "$sub")
+        done
+        for spec in ${WARC_VALIDATE_IP:-}; do
+            revargs+=(--expect-ip "$spec")
+        done
+        for spec in ${WARC_VALIDATE_PROFILE:-}; do
+            revargs+=(--expect-revisit-profile "$spec")
+        done
         test -n "${WARC_VALIDATE_EXCHANGE:-}" && revargs+=(--revisit-exchange)
         info "validating update WARC (revisits)"
         "$python" "$validator" "$(nativepath "$upd")" "${revargs[@]}" >&2 ||
diff --git a/tests/local-server.py b/tests/local-server.py
index ab7c4024..a173a917 100755
--- a/tests/local-server.py
+++ b/tests/local-server.py
@@ -1646,7 +1646,9 @@ def route_bakname_sibling(self):
     ERRMASK_ERR = b"<html><body>error 403</body></html>\n"
 
     def route_errmask_index(self):
-        self.send_html('\t<a href="keep.dat">keep</a>\n')
+        self.send_html(
+            '\t<a href="keep.dat">keep</a>\n\t<a href="empty.dat">empty</a>\n'
+        )
 
     def route_errmask_keep(self):
         # First crawl (no validator) gets the 1024 B body + Last-Modified; the
@@ -1667,6 +1669,24 @@ def route_errmask_keep(self):
         if self.command != "HEAD":
             self.wfile.write(self.ERRMASK_GOOD)
 
+    def route_errmask_empty(self):
+        # Same masking shape as keep.dat, but a genuinely zero-length body: an
+        # engine-forced revisit must still digest and archive it, not treat
+        # it as a missing-crypto case (#839).
+        if self.headers.get("If-Modified-Since") or self.headers.get("If-None-Match"):
+            self.send_response(403, "Forbidden")
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Length", str(len(self.ERRMASK_ERR)))
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(self.ERRMASK_ERR)
+            return
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Last-Modified", BIG_LASTMOD)
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+
     # --- delayed-type degenerate paths (issues #5/#107) --------------------
     def route_delayed_index(self):
         self.send_html(
@@ -2228,6 +2248,7 @@ def route_changes2_ticker(self):
         "/mini304/page.html": route_mini304_page,
         "/errmask/index.html": route_errmask_index,
         "/errmask/keep.dat": route_errmask_keep,
+        "/errmask/empty.dat": route_errmask_empty,
         "/maxrecv/index.html": route_maxrecv_index,
         "/maxrecv/r0.bin": route_maxrecv_404,
         "/maxrecv/r1.bin": route_maxrecv_404,
diff --git a/tests/tests-list.mk b/tests/tests-list.mk
index a51c0879..96d04261 100644
--- a/tests/tests-list.mk
+++ b/tests/tests-list.mk
@@ -201,5 +201,8 @@ TESTS += 108_engine-refetch-backup.test
 TESTS += 122_local-warc-revisit-headers.test
 TESTS += 119_local-proxytrack-ndx-fields2.test
 TESTS += 118_local-proxytrack-arcwrite.test
+TESTS += 134_local-warc-revisit-ip.test
+TESTS += 135_local-warc-forced-notmodified.test
+TESTS += 136_local-warc-cache-priority.test
 TESTS += 137_local-charsetless-encoding.test
 TESTS += 133_engine-reentrant-time.test
diff --git a/tests/warc-validate.py b/tests/warc-validate.py
index fe424030..62d4bf86 100755
--- a/tests/warc-validate.py
+++ b/tests/warc-validate.py
@@ -11,6 +11,12 @@
 #                             WARC-Payload-Digest matching sha1(body) when present
 #   --no-response-for SUB     the asset containing SUB must be a revisit: no
 #                             response may target it, and a revisit must
+#   --no-record-for SUB       no record of any type (response/revisit/resource)
+#                             may target an asset containing SUB
+#   --expect-ip SUB=IP        a response or revisit targeting SUB must carry
+#                             WARC-IP-Address: IP exactly
+#   --expect-revisit-profile SUB=SUBSTR  a revisit targeting SUB must carry a
+#                             WARC-Profile containing SUBSTR
 #   --revisit-exchange        every server-not-modified revisit carries the 304
 #                             exchange it stands for: a 304 status line in its
 #                             block, and a concurrent request record holding the
@@ -109,6 +115,11 @@ def main():
     verbatim = "--verbatim" in argv
     body_specs = [s.split("=", 1) for s in opt_values(argv, "--expect-body-hex")]
     no_resp = opt_values(argv, "--no-response-for")
+    no_record = opt_values(argv, "--no-record-for")
+    ip_specs = [s.split("=", 1) for s in opt_values(argv, "--expect-ip")]
+    profile_specs = [
+        s.split("=", 1) for s in opt_values(argv, "--expect-revisit-profile")
+    ]
     path = [a for a in argv if not a.startswith("--") and "=" not in a][0]
     data = open(path, "rb").read()
 
@@ -121,6 +132,8 @@ def main():
     total = revisits = responses = infos = 0
     body_hits = {sub: False for sub, _ in body_specs}
     revisit_hits = {sub: False for sub in no_resp}
+    ip_hits = {sub: False for sub, _ in ip_specs}
+    profile_hits = {sub: False for sub, _ in profile_specs}
     requests = {}  # WARC-Concurrent-To -> request block
     exchanges = []  # (shown uri, record id) of the server-not-modified revisits
     for rec in records(data):
@@ -145,6 +158,23 @@ def main():
             sys.exit("record %d: missing \\r\\n\\r\\n trailer" % total)
         wtype = field(header, b"WARC-Type")
         uri = field(header, b"WARC-Target-URI") or b""
+        if wtype in (b"response", b"revisit", b"resource"):
+            for sub in no_record:
+                if sub.encode() in uri:
+                    sys.exit(
+                        "unexpected %s record for %s (want no record)"
+                        % (wtype.decode(), sub)
+                    )
+        if wtype in (b"response", b"revisit"):
+            for sub, want_ip in ip_specs:
+                if sub.encode() in uri:
+                    got = field(header, b"WARC-IP-Address") or b""
+                    if got.decode("ascii", "replace") != want_ip:
+                        sys.exit(
+                            "%s for %s: WARC-IP-Address %r != %r"
+                            % (wtype.decode(), sub, got, want_ip)
+                        )
+                    ip_hits[sub] = True
         if wtype == b"warcinfo":
             infos += 1
         elif wtype == b"response":
@@ -183,6 +213,14 @@ def main():
             for sub in no_resp:
                 if sub.encode() in uri:
                     revisit_hits[sub] = True
+            for sub, want_sub in profile_specs:
+                if sub.encode() in uri:
+                    if want_sub.encode() not in profile:
+                        sys.exit(
+                            "revisit for %s: WARC-Profile %r lacks %r"
+                            % (shown, profile, want_sub)
+                        )
+                    profile_hits[sub] = True
             if revisit_exchange and b"server-not-modified" in profile:
                 block = rec[hdr_end : hdr_end + block_len]
                 status = block.split(b"\r\n", 1)[0].split()
@@ -216,6 +254,12 @@ def main():
     for sub, hit in revisit_hits.items():
         if not hit:
             sys.exit("no revisit record found for unchanged asset %s" % sub)
+    for sub, hit in ip_hits.items():
+        if not hit:
+            sys.exit("no response/revisit record found for --expect-ip %s" % sub)
+    for sub, hit in profile_hits.items():
+        if not hit:
+            sys.exit("no revisit record found for --expect-revisit-profile %s" % sub)
     print(
         "warc-validate: %d records OK (%d response, %d revisit)"
         % (total, responses, revisits)