Repository: jhy/jsoup
Pull request #1581 (2021-07-12 15:40:30)

Dependabot: Ignore major Jetty updates
Ignores major updates of Jetty (#1574).


Patch:
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index e3656835..7761dcca 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -5,3 +5,9 @@ updates:
     directory: /
     schedule:
       interval: weekly
+    ignore:
+      # Jetty 9.x needed for JDK8 compatibility; it still receives security updates
+      - dependency-name: "jetty-server"
+        update-types: ["version-update:semver-major"]
+      - dependency-name: "jetty-servlet"
+        update-types: ["version-update:semver-major"]
diff --git a/CHANGES b/CHANGES
index b86ecbf7..262503ea 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,9 @@
 jsoup changelog
 
 *** Release 1.14.2 [PENDING]
+  * Improvement: support Pattern.quote \Q and \E escapes in the selector regex matchers.
+    <https://github.com/jhy/jsoup/pull/1536>
+
   * Bugfix: the *|el wildcard namespace selector now also matches elements with no namespace.
     <https://github.com/jhy/jsoup/issues/1565>
     
@@ -8,10 +11,13 @@ jsoup changelog
 
   * Bugfix: when making a HTTP POST, if the request write fails, make sure the connection is immediately cleaned up.
 
+  * Bugfix: in the XML parser, XML processing instructions without attributes would be serialized as if they did.
+    <https://github.com/jhy/jsoup/issues/770>
+
   * Bugfix: updated the HtmlTreeParser resetInsertionMode to the current spec for supported elements
     <https://github.com/jhy/jsoup/issues/1491>
 
-  * Bugfix [Fuzz]: fixed a slow parse when a tag has thousands of null characters in it.
+  * Bugfix [Fuzz]: fixed a slow parse when a tag or an attribute name has thousands of null characters in it.
     <https://github.com/jhy/jsoup/issues/1580>
 
   * Bugfix [Fuzz]: the adoption agency algorithm can have an incorrect bookmark position
@@ -23,6 +29,10 @@ jsoup changelog
   * Bugfix [Fuzz]: malformed deeply nested table elements could create a stack overflow.
     <https://github.com/jhy/jsoup/issues/1577>
 
+  * Bugfix [Fuzz]: Speed optimized malformed HTML creating elements with thousands of elements - limit the attribute
+    count per element when parsing to 512 (in real-world HTML, P99 is ~ 8).
+    <https://github.com/jhy/jsoup/issues/1578>
+
 *** Release 1.14.1 [2021-Jul-10]
   * Change: updated the minimum supported Java version from Java 7 to Java 8.
 
diff --git a/pom.xml b/pom.xml
index ab146c3c..a3f9fd29 100644
--- a/pom.xml
+++ b/pom.xml
@@ -83,7 +83,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-javadoc-plugin</artifactId>
-        <version>3.1.0</version>
+        <version>3.3.0</version>
         <configuration>
             <doclint>none</doclint>
             <source>7</source>
@@ -120,7 +120,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
-        <version>3.0.2</version>
+        <version>3.2.0</version>
         <configuration>
           <archive>
             <manifestEntries>
@@ -156,7 +156,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-resources-plugin</artifactId>
-        <version>3.0.1</version>
+        <version>3.2.0</version>
       </plugin>
       <plugin>
         <artifactId>maven-release-plugin</artifactId>
@@ -165,7 +165,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>3.0.0-M3</version>
+        <version>3.0.0-M5</version>
         <configuration>
           <!-- smaller stack to find stack overflows -->
           <argLine>-Xss256k</argLine>
@@ -308,7 +308,7 @@
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter</artifactId>
-      <version>5.6.0</version>
+      <version>5.7.2</version>
       <scope>test</scope>
     </dependency>
 
diff --git a/src/main/java/org/jsoup/nodes/Attributes.java b/src/main/java/org/jsoup/nodes/Attributes.java
index b2dc18ef..6e0d7898 100644
--- a/src/main/java/org/jsoup/nodes/Attributes.java
+++ b/src/main/java/org/jsoup/nodes/Attributes.java
@@ -46,6 +46,7 @@ public class Attributes implements Iterable<Attribute>, Cloneable {
     static final int NotFound = -1;
     private static final String EmptyString = "";
 
+    // the number of instance fields is kept as low as possible giving an object size of 24 bytes
     private int size = 0; // number of slots used (not total capacity, which is keys.length)
     String[] keys = new String[InitialCapacity];
     String[] vals = new String[InitialCapacity];
@@ -246,16 +247,12 @@ public class Attributes implements Iterable<Attribute>, Cloneable {
     }
 
     /**
-     Get the number of attributes in this set.
+     Get the number of attributes in this set, including any jsoup internal-only attributes. Internal attributes are
+     excluded from the {@link #html()}, {@link #asList()}, and {@link #iterator()} methods.
      @return size
      */
     public int size() {
-        int s = 0;
-        for (int i = 0; i < size; i++) {
-            if (!isInternalKey(keys[i]))
-                s++;
-        }
-        return s;
+        return size;
     }
 
     /**
diff --git a/src/main/java/org/jsoup/nodes/XmlDeclaration.java b/src/main/java/org/jsoup/nodes/XmlDeclaration.java
index d7906eb6..6ab9b162 100644
--- a/src/main/java/org/jsoup/nodes/XmlDeclaration.java
+++ b/src/main/java/org/jsoup/nodes/XmlDeclaration.java
@@ -52,9 +52,17 @@ public class XmlDeclaration extends LeafNode {
 
     private void getWholeDeclaration(Appendable accum, Document.OutputSettings out) throws IOException {
         for (Attribute attribute : attributes()) {
-            if (!attribute.getKey().equals(nodeName())) { // skips coreValue (name)
+            String key = attribute.getKey();
+            String val = attribute.getValue();
+            if (!key.equals(nodeName())) { // skips coreValue (name)
                 accum.append(' ');
-                attribute.html(accum, out);
+                // basically like Attribute, but skip empty vals in XML
+                accum.append(key);
+                if (!val.isEmpty()) {
+                    accum.append("=\"");
+                    Entities.escape(accum, val, out, true, false, false);
+                    accum.append('"');
+                }
             }
         }
     }
diff --git a/src/main/java/org/jsoup/parser/CharacterReader.java b/src/main/java/org/jsoup/parser/CharacterReader.java
index 35f52fa7..e2297494 100644
--- a/src/main/java/org/jsoup/parser/CharacterReader.java
+++ b/src/main/java/org/jsoup/parser/CharacterReader.java
@@ -555,7 +555,7 @@ public final class CharacterReader {
     }
 
     /**
-     * Caches short strings, as a flywheel pattern, to reduce GC load. Just for this doc, to prevent leaks.
+     * Caches short strings, as a flyweight pattern, to reduce GC load. Just for this doc, to prevent leaks.
      * <p />
      * Simplistic, and on hash collisions just falls back to creating a new string, vs a full HashMap with Entry list.
      * That saves both having to create objects as hash keys, and running through the entry list, at the expense of
@@ -569,27 +569,22 @@ public final class CharacterReader {
             return "";
 
         // calculate hash:
-        int hash = 31 * count;
-        int offset = start;
+        int hash = 0;
         for (int i = 0; i < count; i++) {
-            hash = 31 * hash + charBuf[offset++];
+            hash = 31 * hash + charBuf[start + i];
         }
 
         // get from cache
         final int index = hash & stringCacheSize - 1;
         String cached = stringCache[index];
 
-        if (cached == null) { // miss, add
+        if (cached != null && rangeEquals(charBuf, start, count, cached)) // positive hit
+            return cached;
+        else {
             cached = new String(charBuf, start, count);
-            stringCache[index] = cached;
-        } else { // hashcode hit, check equality
-            if (rangeEquals(charBuf, start, count, cached)) { // hit
-                return cached;
-            } else { // hashcode conflict
-                cached = new String(charBuf, start, count);
-                stringCache[index] = cached; // update the cache, as recently used strings are more likely to show up again
-            }
+            stringCache[index] = cached; // add or replace, assuming most recently used are most likely to recur next
         }
+
         return cached;
     }
 
diff --git a/src/main/java/org/jsoup/parser/Token.java b/src/main/java/org/jsoup/parser/Token.java
index 9ade8e0d..d1b38b91 100644
--- a/src/main/java/org/jsoup/parser/Token.java
+++ b/src/main/java/org/jsoup/parser/Token.java
@@ -97,11 +97,16 @@ abstract class Token {
             return this;
         }
 
+        /* Limits runaway crafted HTML from spewing attributes and getting a little sluggish in ensureCapacity.
+        Real-world HTML will P99 around 8 attributes, so plenty of headroom. Implemented here and not in the Attributes
+        object so that API users can add more if ever required. */
+        private static final int MaxAttributes = 512;
+
         final void newAttribute() {
             if (attributes == null)
                 attributes = new Attributes();
 
-            if (pendingAttributeName != null) {
+            if (pendingAttributeName != null && attributes.size() < MaxAttributes) {
                 // the tokeniser has skipped whitespace control chars, but trimming could collapse to empty for other control codes, so verify here
                 pendingAttributeName = pendingAttributeName.trim();
                 if (pendingAttributeName.length() > 0) {
@@ -176,6 +181,8 @@ abstract class Token {
         }
 
         final void appendAttributeName(String append) {
+            // might have null chars because we eat in one pass - need to replace with null replacement character
+            append = append.replace(TokeniserState.nullChar, Tokeniser.replacementChar);
             pendingAttributeName = pendingAttributeName == null ? append : pendingAttributeName.concat(append);
         }
 
diff --git a/src/main/java/org/jsoup/parser/TokenQueue.java b/src/main/java/org/jsoup/parser/TokenQueue.java
index f2690517..614ade19 100644
--- a/src/main/java/org/jsoup/parser/TokenQueue.java
+++ b/src/main/java/org/jsoup/parser/TokenQueue.java
@@ -38,7 +38,9 @@ public class TokenQueue {
     /**
      * Retrieves but does not remove the first character from the queue.
      * @return First character, or 0 if empty.
+     *@deprecated unused and will be removed in 1.15.x
      */
+    @Deprecated
     public char peek() {
         return isEmpty() ? 0 : queue.charAt(pos);
     }
@@ -46,7 +48,9 @@ public class TokenQueue {
     /**
      Add a character to the start of the queue (will be the next character retrieved).
      @param c character to add
+     @deprecated unused and will be removed in 1.15.x
      */
+    @Deprecated
     public void addFirst(Character c) {
         addFirst(c.toString());
     }
@@ -74,7 +78,9 @@ public class TokenQueue {
      * Case sensitive match test.
      * @param seq string to case sensitively check for
      * @return true if matched, false if not
+     * @deprecated unused and will be removed in 1.15.x
      */
+    @Deprecated
     public boolean matchesCS(String seq) {
         return queue.startsWith(seq, pos);
     }
@@ -104,6 +110,10 @@ public class TokenQueue {
         return false;
     }
 
+    /**
+     @deprecated unused and will be removed in 1.15.x
+     */
+    @Deprecated
     public boolean matchesStartTag() {
         // micro opt for matching "<x"
         return (remainingLength() >= 2 && queue.charAt(pos) == '<' && Character.isLetter(queue.charAt(pos+1)));
@@ -264,6 +274,7 @@ public class TokenQueue {
         char last = 0;
         boolean inSingleQuote = false;
         boolean inDoubleQuote = false;
+        boolean inRegexQE = false; // regex \Q .. \E escapes from Pattern.quote()
 
         do {
             if (isEmpty()) break;
@@ -273,8 +284,10 @@ public class TokenQueue {
                     inSingleQuote = !inSingleQuote;
                 else if (c == '"' && c != open && !inSingleQuote)
                     inDoubleQuote = !inDoubleQuote;
-                if (inSingleQuote || inDoubleQuote)
+                if (inSingleQuote || inDoubleQuote || inRegexQE){
+                    last = c;
                     continue;
+                }
 
                 if (c == open) {
                     depth++;
@@ -283,6 +296,10 @@ public class TokenQueue {
                 }
                 else if (c == close)
                     depth--;
+            } else if (c == 'Q') {
+                inRegexQE = true;
+            } else if (c == 'E') {
+                inRegexQE = false;
             }
 
             if (depth > 0 && last != 0)
@@ -344,7 +361,9 @@ public class TokenQueue {
      * Consume an tag name off the queue (word or :, _, -)
      * 
      * @return tag name
+     * @deprecated unused and will be removed in 1.15.x
      */
+    @Deprecated
     public String consumeTagName() {
         int start = pos;
         while (!isEmpty() && (matchesWord() || matchesAny(':', '_', '-')))
@@ -382,7 +401,9 @@ public class TokenQueue {
     /**
      Consume an attribute key off the queue (letter, digit, -, _, :")
      @return attribute key
+     @deprecated unused and will be removed in 1.15.x
      */
+    @Deprecated
     public String consumeAttributeKey() {
         int start = pos;
         while (!isEmpty() && (matchesWord() || matchesAny('-', '_', ':')))
diff --git a/src/main/java/org/jsoup/parser/TokeniserState.java b/src/main/java/org/jsoup/parser/TokeniserState.java
index e405c1b7..845201fb 100644
--- a/src/main/java/org/jsoup/parser/TokeniserState.java
+++ b/src/main/java/org/jsoup/parser/TokeniserState.java
@@ -144,7 +144,6 @@ enum TokeniserState {
         // from < or </ in data, will have start or end tag pending
         void read(Tokeniser t, CharacterReader r) {
             // previous TagOpen state did NOT consume, will have a letter char in current
-            //String tagName = r.consumeToAnySorted(tagCharsSorted).toLowerCase();
             String tagName = r.consumeTagName();
             t.tagPending.appendTagName(tagName);
 
@@ -608,7 +607,7 @@ enum TokeniserState {
     AttributeName {
         // from before attribute name
         void read(Tokeniser t, CharacterReader r) {
-            String name = r.consumeToAnySorted(attributeNameCharsSorted);
+            String name = r.consumeToAnySorted(attributeNameCharsSorted); // spec deviate - consume and emit nulls in one hit vs stepping
             t.tagPending.appendAttributeName(name);
 
             char c = r.consume();
@@ -630,10 +629,6 @@ enum TokeniserState {
                     t.emitTagPending();
                     t.transition(Data);
                     break;
-                case nullChar:
-                    t.error(this);
-                    t.tagPending.appendAttributeName(replacementChar);
-                    break;
                 case eof:
                     t.eofError(this);
                     t.transition(Data);
@@ -1631,7 +1626,7 @@ enum TokeniserState {
 
     static final char nullChar = '\u0000';
     // char searches. must be sorted, used in inSorted. MUST update TokenisetStateTest if more arrays are added.
-    static final char[] attributeNameCharsSorted = new char[]{nullChar, '\t', '\n', '\f', '\r', ' ', '"', '\'', '/', '<', '=', '>'};
+    static final char[] attributeNameCharsSorted = new char[]{'\t', '\n', '\f', '\r', ' ', '"', '\'', '/', '<', '=', '>'};
     static final char[] attributeValueUnquoted = new char[]{nullChar, '\t', '\n', '\f', '\r', ' ', '"', '&', '\'', '<', '=', '>', '`'};
 
     private static final char replacementChar = Tokeniser.replacementChar;
diff --git a/src/main/java/org/jsoup/select/QueryParser.java b/src/main/java/org/jsoup/select/QueryParser.java
index 3e3b4105..fb2c0797 100644
--- a/src/main/java/org/jsoup/select/QueryParser.java
+++ b/src/main/java/org/jsoup/select/QueryParser.java
@@ -147,7 +147,10 @@ public class QueryParser {
             else if (tq.matches("["))
                 sq.append("[").append(tq.chompBalanced('[', ']')).append("]");
             else if (tq.matchesAny(combinators))
-                break;
+                if (sq.length() > 0)
+                    break;
+                else
+                    tq.consume();
             else
                 sq.append(tq.consume());
         }
diff --git a/src/main/java/org/jsoup/select/Selector.java b/src/main/java/org/jsoup/select/Selector.java
index 1fafbdb3..0afd8c77 100644
--- a/src/main/java/org/jsoup/select/Selector.java
+++ b/src/main/java/org/jsoup/select/Selector.java
@@ -73,6 +73,8 @@ import java.util.IdentityHashMap;
  * <tr><td><code>:empty</code></td><td>elements that have no children at all</td><td></td></tr>
  * </table>
  *
+ * <p>A word on using regular expressions in these selectors: depending on the content of the regex, you will need to quote the pattern using <b><code>Pattern.quote("regex")</code></b> for it to parse correclty through both the selector parser and the regex parser. E.g. <code>String query = "div:matches(" + Pattern.quote(regex) + ");"</code>.</p>
+ *
  * @author Jonathan Hedley, jonathan@hedley.net
  * @see Element#select(String)
  */


Test patch:
diff --git a/src/test/java/org/jsoup/integration/FuzzFixesTest.java b/src/test/java/org/jsoup/integration/FuzzFixesTest.java
index db45b419..122cb59d 100644
--- a/src/test/java/org/jsoup/integration/FuzzFixesTest.java
+++ b/src/test/java/org/jsoup/integration/FuzzFixesTest.java
@@ -104,7 +104,6 @@ public class FuzzFixesTest {
     @Test
     public void overflow1577() throws IOException {
         // https://github.com/jhy/jsoup/issues/1577
-        // no repro - fixed elsewhere?
         File in = ParseTest.getFile("/fuzztests/1577.html.gz");
         Document doc = Jsoup.parse(in, "UTF-8");
         assertNotNull(doc);
@@ -112,4 +111,15 @@ public class FuzzFixesTest {
         Document docXml = Jsoup.parse(new FileInputStream(in), "UTF-8", "https://example.com", Parser.xmlParser());
         assertNotNull(docXml);
     }
+
+    @Test
+    public void parseTimeout36150() throws IOException {
+        File in = ParseTest.getFile("/fuzztests/1580-attrname.html.gz");
+        // pretty much 1MB of null chars in text head
+        Document doc = Jsoup.parse(in, "UTF-8");
+        assertNotNull(doc);
+
+        Document docXml = Jsoup.parse(new FileInputStream(in), "UTF-8", "https://example.com", Parser.xmlParser());
+        assertNotNull(docXml);
+    }
 }
diff --git a/src/test/java/org/jsoup/nodes/AttributesTest.java b/src/test/java/org/jsoup/nodes/AttributesTest.java
index 84d9ed78..d2f986b0 100644
--- a/src/test/java/org/jsoup/nodes/AttributesTest.java
+++ b/src/test/java/org/jsoup/nodes/AttributesTest.java
@@ -225,6 +225,10 @@ public class AttributesTest {
 
         a.put(Attributes.internalKey("baseUri"), "example.com");
         a.put(Attributes.internalKey("another"), "example.com");
-        assertEquals(2, a.size());
+        a.put(Attributes.internalKey("last"), "example.com");
+        a.remove(Attributes.internalKey("last"));
+
+        assertEquals(4, a.size());
+        assertEquals(2, a.asList().size()); // excluded from lists
     }
 }
diff --git a/src/test/java/org/jsoup/parser/TokenQueueTest.java b/src/test/java/org/jsoup/parser/TokenQueueTest.java
index 2690c4bf..463a19e4 100644
--- a/src/test/java/org/jsoup/parser/TokenQueueTest.java
+++ b/src/test/java/org/jsoup/parser/TokenQueueTest.java
@@ -1,8 +1,11 @@
 package org.jsoup.parser;
 
 import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
 import org.junit.jupiter.api.Test;
 
+import java.util.regex.Pattern;
+
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.fail;
 
@@ -95,4 +98,12 @@ public class TokenQueueTest {
             assertEquals("Did not find balanced marker at 'something(or another)) else'", expected.getMessage());
         }
     }
+
+    @Test
+    public void testQuotedPattern() {
+        final Document doc = Jsoup.parse("<div>\\) foo1</div><div>( foo2</div><div>1) foo3</div>");
+        assertEquals("\n\\) foo1",doc.select("div:matches(" + Pattern.quote("\\)") + ")").get(0).childNode(0).toString());
+        assertEquals("\n( foo2",doc.select("div:matches(" + Pattern.quote("(") + ")").get(0).childNode(0).toString());
+        assertEquals("\n1) foo3",doc.select("div:matches(" + Pattern.quote("1)") + ")").get(0).childNode(0).toString());
+    }
 }
diff --git a/src/test/java/org/jsoup/parser/XmlTreeBuilderTest.java b/src/test/java/org/jsoup/parser/XmlTreeBuilderTest.java
index d2527a90..8fb0fbd8 100644
--- a/src/test/java/org/jsoup/parser/XmlTreeBuilderTest.java
+++ b/src/test/java/org/jsoup/parser/XmlTreeBuilderTest.java
@@ -152,6 +152,16 @@ public class XmlTreeBuilderTest {
         assertEquals("<?xml version=\"1\" encoding=\"UTF-8\" something=\"else\"?>", decl.outerHtml());
     }
 
+    @Test
+    public void testParseDeclarationWithoutAttributes() {
+        String xml = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<?myProcessingInstruction My Processing instruction.?>";
+        Document doc = Jsoup.parse(xml, "", Parser.xmlParser());
+        XmlDeclaration decl = (XmlDeclaration) doc.childNode(2);
+        assertEquals("myProcessingInstruction", decl.name());
+        assertTrue(decl.hasAttr("My"));
+        assertEquals("<?myProcessingInstruction My Processing instruction.?>", decl.outerHtml());
+    }
+
     @Test
     public void caseSensitiveDeclaration() {
         String xml = "<?XML version='1' encoding='UTF-8' something='else'?>";
diff --git a/src/test/java/org/jsoup/select/QueryParserTest.java b/src/test/java/org/jsoup/select/QueryParserTest.java
index 34ee8a9f..8c4db234 100644
--- a/src/test/java/org/jsoup/select/QueryParserTest.java
+++ b/src/test/java/org/jsoup/select/QueryParserTest.java
@@ -1,5 +1,7 @@
 package org.jsoup.select;
 
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
 import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.*;
@@ -10,6 +12,19 @@ import static org.junit.jupiter.api.Assertions.*;
  * @author Jonathan Hedley
  */
 public class QueryParserTest {
+
+    @Test public void testConsumeSubQuery() {
+        Document doc = Jsoup.parse("<html><head>h</head><body>" +
+                "<li><strong>l1</strong></li>" +
+                "<a><li><strong>l2</strong></li></a>" +
+                "<p><strong>yes</strong></p>" +
+                "</body></html>");
+        assertEquals("l1 l2 yes", doc.body().select(">p>strong,>*>li>strong").text());
+        assertEquals("l2 yes", doc.select("body>p>strong,body>*>li>strong").text());
+        assertEquals("yes", doc.select(">body>*>li>strong,>body>p>strong").text());
+        assertEquals("l2", doc.select(">body>p>strong,>body>*>li>strong").text());
+    }
+
     @Test public void testOrGetsCorrectPrecedence() {
         // tests that a selector "a b, c d, e f" evals to (a AND b) OR (c AND d) OR (e AND f)"
         // top level or, three child ands