Repository: jhy/jsoup
Pull request #2312 (2025-04-24 22:43:08)

Only match ASCII letters and digits
Replace calls to `java.lang.Character.isLetter()` and `isDigit()`. In all these cases we really only want to allow ASCII letters or digits.

jhy: Cool beans. Could you add a couple positive / negative tests for these, in StringUtils? So that nobody gets a rush of blood to their head and tries to tidy it up back to Character.isLetter.
cketti: Done.

Patch:
diff --git a/src/main/java/org/jsoup/internal/StringUtil.java b/src/main/java/org/jsoup/internal/StringUtil.java
index 10344250..a953a86f 100644
--- a/src/main/java/org/jsoup/internal/StringUtil.java
+++ b/src/main/java/org/jsoup/internal/StringUtil.java
@@ -172,7 +172,7 @@ public final class StringUtil {
     }
 
     /**
-     * Tests if a string is numeric, i.e. contains only digit characters
+     * Tests if a string is numeric, i.e. contains only ASCII digit characters
      * @param string string to test
      * @return true if only digit chars, false if empty or null or contains non-digit chars
      */
@@ -182,7 +182,7 @@ public final class StringUtil {
 
         int l = string.length();
         for (int i = 0; i < l; i++) {
-            if (!Character.isDigit(string.codePointAt(i)))
+            if (!isDigit(string.charAt(i)))
                 return false;
         }
         return true;
@@ -394,4 +394,16 @@ public final class StringUtil {
             },
             StringJoiner::complete);
     }
+
+    public static boolean isAsciiLetter(char c) {
+        return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z';
+    }
+
+    public static boolean isDigit(char c) {
+        return c >= '0' && c <= '9';
+    }
+
+    public static boolean isHexDigit(char c) {
+        return isDigit(c) || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F';
+    }
 }
diff --git a/src/main/java/org/jsoup/parser/CharacterReader.java b/src/main/java/org/jsoup/parser/CharacterReader.java
index 6e340e1d..a7ea5e54 100644
--- a/src/main/java/org/jsoup/parser/CharacterReader.java
+++ b/src/main/java/org/jsoup/parser/CharacterReader.java
@@ -2,6 +2,7 @@ package org.jsoup.parser;
 
 import org.jsoup.helper.Validate;
 import org.jsoup.internal.SoftPool;
+import org.jsoup.internal.StringUtil;
 import org.jspecify.annotations.Nullable;
 
 import java.io.IOException;
@@ -480,11 +481,11 @@ public final class CharacterReader {
         bufferUp();
         int start = bufPos;
         while (bufPos < bufLength) {
-            if (Character.isLetter(charBuf[bufPos])) bufPos++;
+            if (StringUtil.isAsciiLetter(charBuf[bufPos])) bufPos++;
             else break;
         }
         while (!isEmptyNoBufferUp()) {
-            if (isDigit(charBuf[bufPos])) bufPos++;
+            if (StringUtil.isDigit(charBuf[bufPos])) bufPos++;
             else break;
         }
 
@@ -492,7 +493,7 @@ public final class CharacterReader {
     }
 
     String consumeHexSequence() {
-        return consumeMatching(CharacterReader::isHexDigit);
+        return consumeMatching(StringUtil::isHexDigit);
     }
 
     String consumeDigitSequence() {
@@ -556,23 +557,18 @@ public final class CharacterReader {
         return !isEmpty() && Arrays.binarySearch(seq, charBuf[bufPos]) >= 0;
     }
 
-    boolean matchesLetter() {
-        if (isEmpty()) return false;
-        return Character.isLetter(charBuf[bufPos]);
-    }
-
     /**
      Checks if the current pos matches an ascii alpha (A-Z a-z) per https://infra.spec.whatwg.org/#ascii-alpha
      @return if it matches or not
      */
     boolean matchesAsciiAlpha() {
         if (isEmpty()) return false;
-        return isAsciiLetter(charBuf[bufPos]);
+        return StringUtil.isAsciiLetter(charBuf[bufPos]);
     }
 
     boolean matchesDigit() {
         if (isEmpty()) return false;
-        return isDigit(charBuf[bufPos]);
+        return StringUtil.isDigit(charBuf[bufPos]);
     }
 
     boolean matchConsume(String seq) {
@@ -686,17 +682,4 @@ public final class CharacterReader {
     interface CharPredicate {
         boolean test(char c);
     }
-
-    // char predicate functions
-    static boolean isAsciiLetter(char c) {
-        return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z';
-    }
-
-    static boolean isDigit(char c) {
-        return c >= '0' && c <= '9';
-    }
-
-    static boolean isHexDigit(char c) {
-        return isDigit(c) || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F';
-    }
 }
diff --git a/src/main/java/org/jsoup/parser/TokenQueue.java b/src/main/java/org/jsoup/parser/TokenQueue.java
index 6b17beda..026fc24f 100644
--- a/src/main/java/org/jsoup/parser/TokenQueue.java
+++ b/src/main/java/org/jsoup/parser/TokenQueue.java
@@ -310,13 +310,13 @@ public class TokenQueue {
                 out.append(Hyphen_Minus);
 
                 char secondChar = q.current();
-                if (CharacterReader.isDigit(secondChar)) {
+                if (StringUtil.isDigit(secondChar)) {
                     // If the character is the second character and is in the range [0-9] (U+0030 to U+0039) and the
                     // first character is a "-" (U+002D), then the character escaped as code point.
                     appendEscapedCodepoint(out, q.consume());
                 }
             }
-        } else if (CharacterReader.isDigit(firstChar)) {
+        } else if (StringUtil.isDigit(firstChar)) {
             // If the character is the first character and is in the range [0-9] (U+0030 to U+0039), then the character
             // escaped as code point.
             appendEscapedCodepoint(out, q.consume());
@@ -447,11 +447,11 @@ public class TokenQueue {
         }
 
         char firstEscaped = consume();
-        if (!CharacterReader.isHexDigit(firstEscaped)) {
+        if (!StringUtil.isHexDigit(firstEscaped)) {
             out.append(firstEscaped);
         } else {
             reader.unconsume(); // put back the first hex digit
-            String hexString = reader.consumeMatching(CharacterReader::isHexDigit, 6); // consume up to 6 hex digits
+            String hexString = reader.consumeMatching(StringUtil::isHexDigit, 6); // consume up to 6 hex digits
             int codePoint;
             try {
                 codePoint = Integer.parseInt(hexString, 16);
@@ -487,12 +487,12 @@ public class TokenQueue {
 
     // https://www.w3.org/TR/css-syntax-3/#ident-start-code-point
     private static boolean isIdentStart(char c) {
-        return c == '_' || CharacterReader.isAsciiLetter(c) || isNonAscii(c);
+        return c == '_' || StringUtil.isAsciiLetter(c) || isNonAscii(c);
     }
 
     // https://www.w3.org/TR/css-syntax-3/#ident-code-point
     private static boolean isIdent(char c) {
-        return c == Hyphen_Minus || CharacterReader.isDigit(c) || isIdentStart(c);
+        return c == Hyphen_Minus || StringUtil.isDigit(c) || isIdentStart(c);
     }
 
     // https://www.w3.org/TR/css-syntax-3/#newline
diff --git a/src/main/java/org/jsoup/parser/Tokeniser.java b/src/main/java/org/jsoup/parser/Tokeniser.java
index e405e8f5..cb2c740b 100644
--- a/src/main/java/org/jsoup/parser/Tokeniser.java
+++ b/src/main/java/org/jsoup/parser/Tokeniser.java
@@ -197,7 +197,7 @@ final class Tokeniser {
                 reader.matchConsume(prefix);
                 nameRef = prefix;
             }
-            if (inAttribute && (reader.matchesLetter() || reader.matchesDigit() || reader.matchesAny('=', '-', '_'))) {
+            if (inAttribute && (reader.matchesAsciiAlpha() || reader.matchesDigit() || reader.matchesAny('=', '-', '_'))) {
                 // don't want that to match
                 reader.rewindToMark();
                 return null;
diff --git a/src/main/java/org/jsoup/parser/TokeniserState.java b/src/main/java/org/jsoup/parser/TokeniserState.java
index c6992fbc..20e6581e 100644
--- a/src/main/java/org/jsoup/parser/TokeniserState.java
+++ b/src/main/java/org/jsoup/parser/TokeniserState.java
@@ -1197,7 +1197,7 @@ enum TokeniserState {
     },
     DoctypeName {
         @Override void read(Tokeniser t, CharacterReader r) {
-            if (r.matchesLetter()) {
+            if (r.matchesAsciiAlpha()) {
                 String name = r.consumeLetterSequence();
                 t.doctypePending.name.append(name);
                 return;
@@ -1672,7 +1672,7 @@ enum TokeniserState {
      * different else exit transitions.
      */
     private static void handleDataEndTag(Tokeniser t, CharacterReader r, TokeniserState elseTransition) {
-        if (r.matchesLetter()) {
+        if (r.matchesAsciiAlpha()) {
             String name = r.consumeLetterSequence();
             t.tagPending.appendTagName(name);
             t.dataBuffer.append(name);
@@ -1752,7 +1752,7 @@ enum TokeniserState {
     }
 
     private static void handleDataDoubleEscapeTag(Tokeniser t, CharacterReader r, TokeniserState primary, TokeniserState fallback) {
-        if (r.matchesLetter()) {
+        if (r.matchesAsciiAlpha()) {
             String name = r.consumeLetterSequence();
             t.dataBuffer.append(name);
             t.emit(name);


Test patch:
diff --git a/src/test/java/org/jsoup/internal/StringUtilTest.java b/src/test/java/org/jsoup/internal/StringUtilTest.java
index 84cca12d..4e686c46 100644
--- a/src/test/java/org/jsoup/internal/StringUtilTest.java
+++ b/src/test/java/org/jsoup/internal/StringUtilTest.java
@@ -165,4 +165,71 @@ public class StringUtilTest {
         assertFalse(StringUtil.isAscii("测试"));
         assertFalse(StringUtil.isAscii("测试.com"));
     }
+
+    @Test void isAsciiLetter() {
+        assertTrue(StringUtil.isAsciiLetter('a'));
+        assertTrue(StringUtil.isAsciiLetter('n'));
+        assertTrue(StringUtil.isAsciiLetter('z'));
+        assertTrue(StringUtil.isAsciiLetter('A'));
+        assertTrue(StringUtil.isAsciiLetter('N'));
+        assertTrue(StringUtil.isAsciiLetter('Z'));
+
+        assertFalse(StringUtil.isAsciiLetter(' '));
+        assertFalse(StringUtil.isAsciiLetter('-'));
+        assertFalse(StringUtil.isAsciiLetter('0'));
+        assertFalse(StringUtil.isAsciiLetter('ß'));
+        assertFalse(StringUtil.isAsciiLetter('Ě'));
+    }
+
+    @Test void isDigit() {
+        assertTrue(StringUtil.isDigit('0'));
+        assertTrue(StringUtil.isDigit('1'));
+        assertTrue(StringUtil.isDigit('2'));
+        assertTrue(StringUtil.isDigit('3'));
+        assertTrue(StringUtil.isDigit('4'));
+        assertTrue(StringUtil.isDigit('5'));
+        assertTrue(StringUtil.isDigit('6'));
+        assertTrue(StringUtil.isDigit('7'));
+        assertTrue(StringUtil.isDigit('8'));
+        assertTrue(StringUtil.isDigit('9'));
+
+        assertFalse(StringUtil.isDigit('a'));
+        assertFalse(StringUtil.isDigit('A'));
+        assertFalse(StringUtil.isDigit('ä'));
+        assertFalse(StringUtil.isDigit('Ä'));
+        assertFalse(StringUtil.isDigit('١'));
+        assertFalse(StringUtil.isDigit('୳'));
+    }
+
+    @Test void isHexDigit() {
+        assertTrue(StringUtil.isHexDigit('0'));
+        assertTrue(StringUtil.isHexDigit('1'));
+        assertTrue(StringUtil.isHexDigit('2'));
+        assertTrue(StringUtil.isHexDigit('3'));
+        assertTrue(StringUtil.isHexDigit('4'));
+        assertTrue(StringUtil.isHexDigit('5'));
+        assertTrue(StringUtil.isHexDigit('6'));
+        assertTrue(StringUtil.isHexDigit('7'));
+        assertTrue(StringUtil.isHexDigit('8'));
+        assertTrue(StringUtil.isHexDigit('9'));
+        assertTrue(StringUtil.isHexDigit('a'));
+        assertTrue(StringUtil.isHexDigit('b'));
+        assertTrue(StringUtil.isHexDigit('c'));
+        assertTrue(StringUtil.isHexDigit('d'));
+        assertTrue(StringUtil.isHexDigit('e'));
+        assertTrue(StringUtil.isHexDigit('f'));
+        assertTrue(StringUtil.isHexDigit('A'));
+        assertTrue(StringUtil.isHexDigit('B'));
+        assertTrue(StringUtil.isHexDigit('C'));
+        assertTrue(StringUtil.isHexDigit('D'));
+        assertTrue(StringUtil.isHexDigit('E'));
+        assertTrue(StringUtil.isHexDigit('F'));
+
+        assertFalse(StringUtil.isHexDigit('g'));
+        assertFalse(StringUtil.isHexDigit('G'));
+        assertFalse(StringUtil.isHexDigit('ä'));
+        assertFalse(StringUtil.isHexDigit('Ä'));
+        assertFalse(StringUtil.isHexDigit('١'));
+        assertFalse(StringUtil.isHexDigit('୳'));
+    }
 }