Repository: xroche/httrack (2026-07-27T20:22:46Z)

Issue:
A query-string character reference the page charset cannot represent is left unescaped, changing the query's parse
On a document with no declared charset (fallback iso-8859-1), HTTrack leaves an HTML character reference that iso-8859-1 cannot represent as literal source text in the query string, instead of percent-encoding it. The bare `&` and `#` then act as query delimiters, so the URL HTTrack records and replays parses differently from the one the document expressed.

Reproducer, page served as `text/html` with no charset:

    <a href="q.php?a=&euro;&c=&eacute;">q</a>

Request sent:

    GET /q.php?a=&euro;&c=%e9 HTTP/1.1

Control, same markup on a page declaring `charset=utf-8`:

    GET /q.php?a=%e2%82%ac&c=%c3%a9 HTTP/1.1

The origin sees three parameters in the first case (`a=` empty, a bare `euro;` flag, `c=%e9`) where the document expressed two, `a=€` and `c=é`. `&eacute;` comes through because iso-8859-1 covers it; `&euro;` is the one that falls through.

The URL Standard's percent-encode after encoding algorithm requires a code point the encoding cannot represent to be written as `%26%23`, the code point in decimal, then `%3B`. Under iso-8859-1 that makes `€` into `%26%238364%3B`, never a bare `&#8364;` and never an undecoded `&euro;`.

`hts_unescapeEntitiesWithCharset` decodes the entity, fails to convert it through `hts_convertStringFromUTF8`, leaves `len` at 0 and falls through to copying the raw source bytes: src/htsencoding.c:135-140, reached from src/htsparse.c:2157-2164.

Direction: emit `%26%23<decimal>%3B` when the conversion fails. Nothing in the tree does this today; `hts_convertStringFromUTF8` has one engine caller.



Discussion:

Patch:
diff --git a/src/htscharset.c b/src/htscharset.c
index e9264deb..c36cf790 100644
--- a/src/htscharset.c
+++ b/src/htscharset.c
@@ -296,16 +296,41 @@ LPWSTR hts_convertUTF8StringToUCS2(const char *s, int size, int *pwsize) {
   return hts_convertStringToUCS2(s, size, CP_UTF8, pwsize);
 }
 
-char *hts_convertUCS2StringToCP(LPWSTR woutput, int wsize, UINT cp) {
+/* WideCharToMultiByte rejects lpUsedDefaultChar on the Unicode, ISO-2022, HZ,
+   GB18030 and ISCII codepages, where a substitution stays invisible. */
+static hts_boolean cp_reports_default_char(UINT cp) {
+  if (cp == 42 /* CP_SYMBOL */ || cp == CP_UTF7 || cp == CP_UTF8 ||
+      cp == 52936 || cp == 54936 || (cp >= 50220 && cp <= 50229) ||
+      (cp >= 57002 && cp <= 57011)) {
+    return HTS_FALSE;
+  }
+  return HTS_TRUE;
+}
+
+/* When plossy is non-NULL, *plossy reports that the codepage lacked a character
+   and a substitute was emitted for it. */
+static char *hts_convertUCS2StringToCPEx(LPWSTR woutput, int wsize, UINT cp,
+                                         hts_boolean *plossy) {
   const int usize =
-    WideCharToMultiByte(cp, 0, woutput, wsize, NULL, 0, NULL, FALSE);
+      WideCharToMultiByte(cp, 0, woutput, wsize, NULL, 0, NULL, NULL);
+
+  if (plossy != NULL) {
+    *plossy = HTS_FALSE;
+  }
   if (usize > 0) {
     char *const uoutput = malloc((usize + 1) * sizeof(char));
 
     if (uoutput != NULL) {
-      if (WideCharToMultiByte
-          (cp, 0, woutput, wsize, uoutput, usize, NULL, FALSE) == usize) {
+      BOOL usedDefault = FALSE;
+      LPBOOL const pUsedDefault =
+          plossy != NULL && cp_reports_default_char(cp) ? &usedDefault : NULL;
+
+      if (WideCharToMultiByte(cp, 0, woutput, wsize, uoutput, usize, NULL,
+                              pUsedDefault) == usize) {
         uoutput[usize] = '\0';
+        if (plossy != NULL && usedDefault) {
+          *plossy = HTS_TRUE;
+        }
         return uoutput;
       } else {
         free(uoutput);
@@ -315,6 +340,10 @@ char *hts_convertUCS2StringToCP(LPWSTR woutput, int wsize, UINT cp) {
   return NULL;
 }
 
+char *hts_convertUCS2StringToCP(LPWSTR woutput, int wsize, UINT cp) {
+  return hts_convertUCS2StringToCPEx(woutput, wsize, cp, NULL);
+}
+
 char *hts_convertUCS2StringToUTF8(LPWSTR woutput, int wsize) {
   return hts_convertUCS2StringToCP(woutput, wsize, CP_UTF8);
 }
@@ -346,7 +375,11 @@ char *hts_convertStringCPToUTF8(const char *s, size_t size, UINT cp) {
   return NULL;
 }
 
-char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
+static char *hts_convertStringCPFromUTF8Ex(const char *s, size_t size, UINT cp,
+                                           hts_boolean *plossy) {
+  if (plossy != NULL) {
+    *plossy = HTS_FALSE;
+  }
   /* Empty string ? */
   if (size == 0) {
     return hts_stringMemCopy(s, size);
@@ -362,7 +395,8 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
     LPWSTR woutput = hts_convertStringToUCS2(s, (int) size, CP_UTF8, &wsize);
 
     if (woutput != NULL) {
-      char *const uoutput = hts_convertUCS2StringToCP(woutput, wsize, cp);
+      char *const uoutput =
+          hts_convertUCS2StringToCPEx(woutput, wsize, cp, plossy);
 
       free(woutput);
       return uoutput;
@@ -373,6 +407,10 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
   return NULL;
 }
 
+char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
+  return hts_convertStringCPFromUTF8Ex(s, size, cp, NULL);
+}
+
 HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
                                          const char *charset) {
   const UINT cp = hts_getCodepage(charset);
@@ -386,6 +424,19 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
   return hts_convertStringCPFromUTF8(s, size, cp);
 }
 
+char *hts_convertStringFromUTF8Strict(const char *s, size_t size,
+                                      const char *charset) {
+  hts_boolean lossy = HTS_FALSE;
+  char *const out =
+      hts_convertStringCPFromUTF8Ex(s, size, hts_getCodepage(charset), &lossy);
+
+  if (lossy) {
+    free(out);
+    return NULL;
+  }
+  return out;
+}
+
 HTSEXT_API char *hts_convertStringSystemToUTF8(const char *s, size_t size) {
   return hts_convertStringCPToUTF8(s, size, GetACP());
 }
@@ -590,6 +641,13 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
   }
 }
 
+char *hts_convertStringFromUTF8Strict(const char *s, size_t size,
+                                      const char *charset) {
+  /* No transliteration is requested of iconv, so an unrepresentable code point
+     already fails the conversion outright. */
+  return hts_convertStringFromUTF8(s, size, charset);
+}
+
 #endif
 
 #ifdef _WIN32
diff --git a/src/htscharset.h b/src/htscharset.h
index b162f8f9..90781553 100644
--- a/src/htscharset.h
+++ b/src/htscharset.h
@@ -61,6 +61,15 @@ HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
 extern char *hts_convertStringFromUTF8(const char *s, size_t size,
                                        const char *charset);
 
+/**
+ * Same, but refusing to lose a code point the charset can not represent:
+ * NULL is returned then, where hts_convertStringFromUTF8() may hand back a
+ * substituted string (Windows substitutes, iconv fails).
+ * Return NULL upon error.
+ **/
+extern char *hts_convertStringFromUTF8Strict(const char *s, size_t size,
+                                             const char *charset);
+
 /**
  * Convert an UTF-8 string to an IDNA (RFC 3492) string.
  **/
diff --git a/src/htsencoding.c b/src/htsencoding.c
index d15e006b..cd7ca396 100644
--- a/src/htsencoding.c
+++ b/src/htsencoding.c
@@ -63,6 +63,13 @@ static int get_hex_value(char c) {
   } while (0)
 
 int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t max, const char *charset) {
+  return hts_unescapeEntitiesWithCharsetSpecial(src, dest, max, charset, 0);
+}
+
+int hts_unescapeEntitiesWithCharsetSpecial(const char *src, char *dest,
+                                           const size_t max,
+                                           const char *charset,
+                                           const int flags) {
   size_t i, j, ampStart, ampStartDest;
   int uc;
   int hex;
@@ -120,9 +127,18 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
             char buffer[32];
             len = 0;
             if ( ( ulen = hts_writeUTF8(uc, buffer, sizeof(buffer)) ) != 0) {
+              const hts_boolean urlQuery =
+                  (flags & UNESCAPE_ENTITIES_URL_QUERY) != 0;
               char *s;
               buffer[ulen] = '\0';
-              s = hts_convertStringFromUTF8(buffer, strlen(buffer), charset);
+              /* Strict for a query only: a substituted '?' must not pass for
+                 the code point the document wrote. */
+              if (urlQuery) {
+                s = hts_convertStringFromUTF8Strict(buffer, strlen(buffer),
+                                                    charset);
+              } else {
+                s = hts_convertStringFromUTF8(buffer, strlen(buffer), charset);
+              }
               if (s != NULL) {
                 const size_t sLen = strlen(s);
                 if (sLen < maxOut) {
@@ -130,7 +146,18 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
                   memcpy(&dest[ampStartDest], s, sLen);
                   len = sLen;
                 }
-                free(s);
+                freet(s);
+              } else if (urlQuery) {
+                /* URL Standard: an unrepresentable code point is written
+                   %26%23<decimal>%3B rather than left as source text. */
+                char esc[32];
+                const int escLen =
+                    snprintf(esc, sizeof(esc), "%%26%%23%d%%3B", uc);
+
+                if (escLen > 0 && (size_t) escLen < maxOut) {
+                  memcpy(&dest[ampStartDest], esc, (size_t) escLen);
+                  len = (size_t) escLen;
+                }
               }
             }
           }
diff --git a/src/htsencoding.h b/src/htsencoding.h
index 369cf833..3f17c02c 100644
--- a/src/htsencoding.h
+++ b/src/htsencoding.h
@@ -69,6 +69,29 @@ extern int hts_unescapeEntitiesWithCharset(const char *src,
                                            char *dest, const size_t max,
                                            const char *charset);
 
+/**
+ * Flags for hts_unescapeEntitiesWithCharsetSpecial(). Values stay distinct from
+ * unescapeFlags above: both reach their function as a plain int.
+ **/
+typedef enum unescapeEntitiesFlags {
+  /** The destination is a URL query string: write a reference the charset can
+      not represent as %26%23<decimal>%3B (URL Standard), instead of source text
+      whose '&' and '#' would re-split the query. **/
+  UNESCAPE_ENTITIES_URL_QUERY = 2
+} unescapeEntitiesFlags;
+
+/**
+ * Unescape HTML entities into their charset equivalents, "flags" being a mask
+ * of UNESCAPE_ENTITIES_XXX constants.
+ * Note: source and destination MUST NOT be the same with a flag that may grow
+ * the string (UNESCAPE_ENTITIES_URL_QUERY).
+ * Returns 0 upon success, -1 upon overflow or error.
+ **/
+extern int hts_unescapeEntitiesWithCharsetSpecial(const char *src, char *dest,
+                                                  const size_t max,
+                                                  const char *charset,
+                                                  const int flags);
+
 /**
  * Unescape an URL-encoded string. The implicit charset is UTF-8.
  * In case of UTF-8 decoding error inside URL-encoded characters, 
diff --git a/src/htsparse.c b/src/htsparse.c
index f057d101..97b53c38 100644
--- a/src/htsparse.c
+++ b/src/htsparse.c
@@ -2176,13 +2176,20 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       "could not decode URI '%s' with charset '%s'", lien, charset);
                   }
 
-                  // decode query string entities with page charset
+                  // decode query entities with the page charset, out-of-place
+                  // because the escape grows it (#854)
                   if (hasCharset) {
-                    if (hts_unescapeEntitiesWithCharset(query, 
-                                                        query, strlen(query) + 1,
-                                                        charset) != 0) {
-                        hts_log_print(opt, LOG_WARNING,
-                          "could not decode query string '%s' with charset '%s'", query, charset);
+                    char BIGSTK decoded[sizeof(query)];
+
+                    if (hts_unescapeEntitiesWithCharsetSpecial(
+                            query, decoded, sizeof(decoded), charset,
+                            UNESCAPE_ENTITIES_URL_QUERY) == 0) {
+                      strcpybuff(query, decoded);
+                    } else {
+                      hts_log_print(opt, LOG_WARNING,
+                                    "could not decode query string '%s' with "
+                                    "charset '%s'",
+                                    query, charset);
                     }
                   }
 


Test patch:
diff --git a/tests/139_local-query-charref.test b/tests/139_local-query-charref.test
new file mode 100755
index 00000000..f7a2b8e0
--- /dev/null
+++ b/tests/139_local-query-charref.test
@@ -0,0 +1,127 @@
+#!/bin/bash
+# Issue #854: a character reference the page charset can not represent must be
+# percent-escaped in a query string (%26%23<decimal>%3B), not left as source
+# text whose '&' and '#' split the query into extra parameters. A reference the
+# charset does represent must still go out as its encoded bytes.
+set -eu
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+server=$(nativepath "${testdir}/local-server.py")
+root=$(nativepath "${testdir}/server-root")
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+httrack=$(command -v httrack) || ! echo "could not find httrack" >&2 || exit 1
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_854.XXXXXX") || exit 1
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'cleanup || true' EXIT HUP INT QUIT PIPE TERM
+
+serverlog="${tmpdir}/server.log"
+: >"$serverlog"
+reqlog="${tmpdir}/requests.txt"
+: >"$reqlog"
+CHARREF_LOG="$reqlog" "$python" "$server" --root "$root" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 50); do
+    line=$(head -n1 "$serverlog" 2>/dev/null) || line=
+    if test "${line%% *}" == "PORT"; then
+        port="${line#PORT }"
+        break
+    fi
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "server exited early: $(cat "$serverlog")"
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover server port"
+    exit 1
+}
+
+"$httrack" -O "${tmpdir}/crawl" --quiet --disable-security-limits --robots=0 \
+    --timeout=30 --retries=0 "http://127.0.0.1:${port}/charref/index.html" \
+    >"${tmpdir}/crawl.log" 2>&1 || true
+
+query_of() {
+    grep -F "/charref/$1.html?" "$reqlog" | cut -f1 |
+        sed "s|^/charref/$1.html?||" | sort -u
+}
+
+# check <label> <page> <want-parameter-count> <acceptable-query>...
+# The count is the origin's own parse of the wire bytes, independent of the
+# expected string. More than one query is acceptable where iconv and Windows
+# encode the same code point differently.
+check() {
+    local label="$1" page="$2" wantcount="$3" got count want ok=
+    shift 3
+    printf '[%s] ..\t' "$label"
+    got=$(query_of "$page") || got=
+    count=$(grep -F "/charref/${page}.html?" "$reqlog" | cut -f2 | sort -u) || count=
+    for want in "$@"; do
+        if test "$got" == "$want"; then
+            ok=1
+        fi
+    done
+    if test -z "$ok"; then
+        echo "FAIL: expected query '$1', got '${got}'"
+        exit 1
+    fi
+    if test "$count" != "$wantcount"; then
+        echo "FAIL: origin parsed '${count}' parameters, expected ${wantcount}"
+        exit 1
+    fi
+    echo "OK (${count} parameters)"
+}
+
+# iso-8859-1 fallback: € is unrepresentable (named and numeric alike), é is not.
+check "charset-less page" qnone 4 \
+    'a=%26%238364%3B&b=%26%238364%3B&c=%e9&d=x'
+# Control: utf-8 represents both, so they go out as their encoded bytes.
+check "declared utf-8 page" qutf8 4 \
+    'a=%e2%82%ac&b=%e2%82%ac&c=%c3%a9&d=x'
+# Multi-byte: あ and 〜 must survive as their encoded bytes, only € gets escaped.
+# The two cp932 tables disagree about 〜 (U+301C): iconv encodes it, Windows maps
+# 0x8160 to U+FF5E instead and reports the substitution, so both are correct.
+check "declared shift_jis page" qsjis 4 \
+    'a=%82%a0&c=%81`&b=%26%238364%3B&d=x' \
+    'a=%82%a0&c=%26%2312316%3B&b=%26%238364%3B&d=x'
+
+# iso-2022-jp encodes both references 7-bit clean, where a round-trip check saw
+# every one of them as unrepresentable. Asserted per field: iconv and Windows
+# differ on the trailing shift back to ASCII, so the bytes are not one string.
+printf '[declared iso-2022-jp page] ..\t'
+jis=$(query_of qjis) || jis=
+jiscount=$(grep -F "/charref/qjis.html?" "$reqlog" | cut -f2 | sort -u) || jiscount=
+for field in ${jis//&/ }; do
+    case "$field" in
+    a=%1b\$B* | c=%1b\$B*)
+        case "$field" in
+        *%26%23*)
+            echo "FAIL: '$field' escaped a reference iso-2022-jp represents"
+            exit 1
+            ;;
+        esac
+        ;;
+    # WideCharToMultiByte refuses lpUsedDefaultChar on a stateful codepage, so
+    # Windows substitutes € invisibly; harmless, '?' is no query delimiter.
+    b=%26%238364%3B | b=%3f | 'b=?' | d=x) ;;
+    *)
+        echo "FAIL: unexpected field '$field' in '$jis'"
+        exit 1
+        ;;
+    esac
+done
+if test "$jiscount" != 4; then
+    echo "FAIL: origin parsed '${jiscount}' parameters, expected 4"
+    exit 1
+fi
+echo "OK (${jiscount} parameters)"
diff --git a/tests/local-server.py b/tests/local-server.py
index ab7c4024..72df9706 100755
--- a/tests/local-server.py
+++ b/tests/local-server.py
@@ -1196,6 +1196,45 @@ def route_charset(self):
         self.send_header("Content-Length", "0")
         self.end_headers()
 
+    # Character references in a query string (#854). CHARREF_LOG collects the
+    # request-target the origin saw, plus the parameter count it parses out of
+    # it. Per page: a reference the declared charset represents (é, あ) and one
+    # it does not (€).
+    CHARREF_QUERY = "?a=&euro;&b=&#8364;&c=&eacute;&d=x"
+    CHARREF_MB_QUERY = "?a=&#12354;&c=&#12316;&b=&euro;&d=x"
+    CHARREF_PAGES = {
+        "none": ("text/html", CHARREF_QUERY),
+        "utf8": ("text/html; charset=utf-8", CHARREF_QUERY),
+        "sjis": ("text/html; charset=shift_jis", CHARREF_MB_QUERY),
+        "jis": ("text/html; charset=iso-2022-jp", CHARREF_MB_QUERY),
+    }
+
+    def record_charref(self):
+        log = os.environ.get("CHARREF_LOG")
+        if log:
+            fields = [f for f in urlsplit(self.path).query.split("&") if f]
+            with open(log, "a") as fp:
+                fp.write("%s\t%d\n" % (self.path, len(fields)))
+
+    def route_charref(self):
+        name = urlsplit(self.path).path[len("/charref/") :]
+        variant = name[: -len(".html")] if name.endswith(".html") else ""
+        if variant == "index":
+            body = "".join(
+                '<a href="%s.html">%s</a>' % (v, v) for v in self.CHARREF_PAGES
+            )
+            self.send_raw(("<html><body>%s</body></html>" % body).encode(), "text/html")
+        elif variant in self.CHARREF_PAGES:
+            ctype, query = self.CHARREF_PAGES[variant]
+            body = '<a href="q%s.html%s">q</a>' % (variant, query)
+            self.send_raw(("<html><body>%s</body></html>" % body).encode(), ctype)
+        elif variant[1:] in self.CHARREF_PAGES:
+            self.send_raw(b"<html><body>q</body></html>", "text/html")
+        else:
+            self.send_response(404)
+            self.send_header("Content-Length", "0")
+            self.end_headers()
+
     # variant -> (Content-Type, title bytes, body bytes); the gb2312 and l1decl
     # titles are valid UTF-8 as bytes, so only the declared charset decodes them
     TITLEENC_PAGES = {
@@ -2415,6 +2454,9 @@ def dispatch(self):
         if path.startswith("/charset/"):
             self.route_charset()
             return True
+        if path.startswith("/charref/"):
+            self.route_charref()
+            return True
         if path.startswith("/titleenc/"):
             self.route_titleenc()
             return True
@@ -2433,6 +2475,9 @@ def dispatch(self):
         return False
 
     def do_GET(self):
+        # Before reject_fragment(), so an unescaped '#' still shows up recorded.
+        if self.path.startswith("/charref/"):
+            self.record_charref()
         if self.reject_fragment():
             return
         if not self.dispatch():
diff --git a/tests/tests-list.mk b/tests/tests-list.mk
index a51c0879..8494d031 100644
--- a/tests/tests-list.mk
+++ b/tests/tests-list.mk
@@ -203,3 +203,4 @@ TESTS += 119_local-proxytrack-ndx-fields2.test
 TESTS += 118_local-proxytrack-arcwrite.test
 TESTS += 137_local-charsetless-encoding.test
 TESTS += 133_engine-reentrant-time.test
+TESTS += 139_local-query-charref.test