Repository: jhy/jsoup
Pull request #2114 (2024-02-01 11:25:35)

Fix NullPointerException in case a custom document is passed to W3CDom
When we provide a custom output `Document` to `W3CDom#convert`, we receive a `NullPointerException` in the `W3CBuilder` constructor.

I've added a test that demonstrates the behavior. Here's the stacktrace from that test without the applied fix.

```
java.lang.NullPointerException: Cannot invoke "org.jsoup.nodes.Element.ownerDocument()" because "this.contextElement" is null

	at org.jsoup.helper.W3CDom$W3CBuilder.<init>(W3CDom.java:364)
	at org.jsoup.helper.W3CDom.convert(W3CDom.java:250)
	at org.jsoup.helper.W3CDom.convert(W3CDom.java:238)
	at org.jsoup.helper.W3CDomTest.canConvertToCustomDocument(W3CDomTest.java:199)
```

casid: Hi @jhy,

did you get a chance to have a look at this already?

It is not urgent, but it prevents us from upgrading to the latest JSOUP version without hacking the classpass.

Thank you so much!

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 8f0d8a49..8a76fd2c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -6,11 +6,22 @@
 
 * Added `Path` accepting parse methods: `Jsoup.parse(Path)`, `Jsoup.parse(path, charsetName, baseUri, parser)`,
   etc. [2055](https://github.com/jhy/jsoup/pull/2055)
+* Updated the `button` tag configuration to include a space between multiple button elements in the `Element.text()` 
+  method. [2105](https://github.com/jhy/jsoup/issues/2105)
 
 ### Changes
 
 * Removed previously deprecated internal classes and methods. [2094](https://github.com/jhy/jsoup/pull/2094)
 
+### Bug Fixes
+
+* When tracking source positions, if the first node was a TextNode, its position was incorrectly set
+  to `-1.` [2106](https://github.com/jhy/jsoup/issues/2106)
+* When connecting (or redirecting) to URLs with characters such as `{`, `}` in the path, a Malformed URL exception would
+  be thrown (if in development), or the URL might otherwise not be escaped correctly (if in
+  production). The URL encoding process has been improved to handle these characters
+  correctly. [2142](https://github.com/jhy/jsoup/issues/2142)
+
 ---
 
 ## 1.17.2 (2023-Dec-29)
diff --git a/pom.xml b/pom.xml
index e25abaec..daa777a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -228,7 +228,7 @@
         <!-- API version compat check - https://siom79.github.io/japicmp/ -->
         <groupId>com.github.siom79.japicmp</groupId>
         <artifactId>japicmp-maven-plugin</artifactId>
-        <version>0.18.3</version>
+        <version>0.20.0</version>
         <configuration>
           <!-- hard code previous version; can't detect when running stateless on build server -->
           <oldVersion>
@@ -404,7 +404,7 @@
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter</artifactId>
-      <version>5.10.1</version>
+      <version>5.10.3</version>
       <scope>test</scope>
     </dependency>
 
diff --git a/src/main/java/org/jsoup/helper/UrlBuilder.java b/src/main/java/org/jsoup/helper/UrlBuilder.java
index 30213d37..b0e9d13a 100644
--- a/src/main/java/org/jsoup/helper/UrlBuilder.java
+++ b/src/main/java/org/jsoup/helper/UrlBuilder.java
@@ -81,15 +81,22 @@ final class UrlBuilder {
         }
     }
 
+    private static final String unsafeCharacters = "<>\"{}|\\^[]`";
+
     private static void appendToAscii(String s, boolean spaceAsPlus, StringBuilder sb) throws UnsupportedEncodingException {
-        // minimal normalization of Unicode -> Ascii, and space normal. Existing escapes are left as-is.
         for (int i = 0; i < s.length(); i++) {
             int c = s.codePointAt(i);
             if (c == ' ') {
                 sb.append(spaceAsPlus ? '+' : "%20");
-            } else if (c > 127) { // out of ascii range
+            } else if (c == '%') { // if already a valid escape, pass; otherwise, escape
+                if (i < s.length() - 2 && isHex(s.charAt(i + 1)) && isHex(s.charAt(i + 2))) {
+                    sb.append('%').append(s.charAt(i + 1)).append(s.charAt(i + 2));
+                    i += 2; // skip the next two characters
+                } else {
+                    sb.append("%25");
+                }
+            } else if (c > 127 || unsafeCharacters.indexOf(c) != -1) { // past ascii, or otherwise unsafe
                 sb.append(URLEncoder.encode(new String(Character.toChars(c)), UTF_8.name()));
-                // ^^ is a bit heavy-handed - if perf critical, we could optimize
                 if (Character.charCount(c) == 2) i++; // advance past supplemental
             } else {
                 sb.append((char) c);
@@ -97,5 +104,9 @@ final class UrlBuilder {
         }
     }
 
+    private static boolean isHex(char c) {
+        return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f');
+    }
+
 
 }
diff --git a/src/main/java/org/jsoup/helper/W3CDom.java b/src/main/java/org/jsoup/helper/W3CDom.java
index 6ce0abd9..3946366e 100644
--- a/src/main/java/org/jsoup/helper/W3CDom.java
+++ b/src/main/java/org/jsoup/helper/W3CDom.java
@@ -360,12 +360,14 @@ public class W3CDom {
             namespacesStack.push(new HashMap<>());
             dest = doc;
             contextElement = (org.jsoup.nodes.Element) doc.getUserData(ContextProperty); // Track the context jsoup Element, so we can save the corresponding w3c element
-            final org.jsoup.nodes.Document inDoc = contextElement.ownerDocument();
-            if (namespaceAware && inDoc != null && inDoc.parser().getTreeBuilder() instanceof HtmlTreeBuilder) {
-              // as per the WHATWG HTML5 spec § 2.1.3, elements are in the HTML namespace by default
-              namespacesStack.peek().put("", Parser.NamespaceHtml);
+            if (contextElement != null) {
+                final org.jsoup.nodes.Document inDoc = contextElement.ownerDocument();
+                if ( namespaceAware && inDoc != null && inDoc.parser().getTreeBuilder() instanceof HtmlTreeBuilder ) {
+                    // as per the WHATWG HTML5 spec § 2.1.3, elements are in the HTML namespace by default
+                    namespacesStack.peek().put("", Parser.NamespaceHtml);
+                }
             }
-          }
+        }
 
         public void head(org.jsoup.nodes.Node source, int depth) {
             namespacesStack.push(new HashMap<>(namespacesStack.peek())); // inherit from above on the stack
diff --git a/src/main/java/org/jsoup/parser/Tag.java b/src/main/java/org/jsoup/parser/Tag.java
index abac6e8f..26f0aac0 100644
--- a/src/main/java/org/jsoup/parser/Tag.java
+++ b/src/main/java/org/jsoup/parser/Tag.java
@@ -271,7 +271,7 @@ public class Tag implements Cloneable {
     private static final String[] inlineTags = {
             "object", "base", "font", "tt", "i", "b", "u", "big", "small", "em", "strong", "dfn", "code", "samp", "kbd",
             "var", "cite", "abbr", "time", "acronym", "mark", "ruby", "rt", "rp", "rtc", "a", "img", "br", "wbr", "map", "q",
-            "sub", "sup", "bdo", "iframe", "embed", "span", "input", "select", "textarea", "label", "button", "optgroup",
+            "sub", "sup", "bdo", "iframe", "embed", "span", "input", "select", "textarea", "label", "optgroup",
             "option", "legend", "datalist", "keygen", "output", "progress", "meter", "area", "param", "source", "track",
             "summary", "command", "device", "area", "basefont", "bgsound", "menuitem", "param", "source", "track",
             "data", "bdi", "s", "strike", "nobr",
@@ -286,7 +286,7 @@ public class Tag implements Cloneable {
     // todo - rework this to format contents as inline; and update html emitter in Element. Same output, just neater.
     private static final String[] formatAsInlineTags = {
             "title", "a", "p", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "address", "li", "th", "td", "script", "style",
-            "ins", "del", "s"
+            "ins", "del", "s", "button"
     };
     private static final String[] preserveWhitespaceTags = {
             "pre", "plaintext", "title", "textarea"
diff --git a/src/main/java/org/jsoup/parser/Tokeniser.java b/src/main/java/org/jsoup/parser/Tokeniser.java
index 1d7b3ff6..2416a66d 100644
--- a/src/main/java/org/jsoup/parser/Tokeniser.java
+++ b/src/main/java/org/jsoup/parser/Tokeniser.java
@@ -50,7 +50,7 @@ final class Tokeniser {
     @Nullable private String lastStartCloseSeq; // "</" + lastStartTag, so we can quickly check for that in RCData
 
     private static final int Unset = -1;
-    private int markupStartPos, charStartPos = Unset; // reader pos at the start of markup / characters. updated on state transition
+    private int markupStartPos, charStartPos = 0; // reader pos at the start of markup / characters. updated on state transition. Initialized to start (0), but set to Unset after emissions.
 
     Tokeniser(TreeBuilder treeBuilder) {
         tagPending = startPending  = new Token.StartTag(treeBuilder);


Test patch:
diff --git a/src/test/java/org/jsoup/helper/HttpConnectionTest.java b/src/test/java/org/jsoup/helper/HttpConnectionTest.java
index 5757d1d2..833fead4 100644
--- a/src/test/java/org/jsoup/helper/HttpConnectionTest.java
+++ b/src/test/java/org/jsoup/helper/HttpConnectionTest.java
@@ -280,8 +280,8 @@ public class HttpConnectionTest {
         URL url1 = new URL("https://test.com/[foo] bar+/%5BOne%5D?q=white space#frag ment");
         URL url2 = new UrlBuilder(url1).build();
         URL url3 = new UrlBuilder(url2).build();
-        assertEquals("https://test.com/[foo]%20bar+/%5BOne%5D?q=white+space#frag%20ment", url2.toExternalForm());
-        assertEquals("https://test.com/[foo]%20bar+/%5BOne%5D?q=white+space#frag%20ment", url3.toExternalForm());
+        assertEquals("https://test.com/%5Bfoo%5D%20bar+/%5BOne%5D?q=white+space#frag%20ment", url2.toExternalForm());
+        assertEquals("https://test.com/%5Bfoo%5D%20bar+/%5BOne%5D?q=white+space#frag%20ment", url3.toExternalForm());
     }
 
     @Test void connectToEncodedUrl() {
diff --git a/src/test/java/org/jsoup/helper/W3CDomTest.java b/src/test/java/org/jsoup/helper/W3CDomTest.java
index de0ce166..e236b29f 100644
--- a/src/test/java/org/jsoup/helper/W3CDomTest.java
+++ b/src/test/java/org/jsoup/helper/W3CDomTest.java
@@ -13,6 +13,7 @@ import org.xml.sax.InputSource;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.OutputKeys;
 import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathExpression;
@@ -187,6 +188,20 @@ public class W3CDomTest {
         assertEquals("<?xml version=\"1.0\" encoding=\"UTF-8\"?><html xmlns=\"http://www.w3.org/1999/xhtml\"><head/><body>&lt;インセンティブで高収入！&gt;Text <p>More</p></body></html>", xml);
     }
 
+    @Test
+    public void canConvertToCustomDocument() throws ParserConfigurationException {
+        org.jsoup.nodes.Document document = Jsoup.parse("<html><div></div></html>");
+
+        DocumentBuilderFactory localDocumentBuilderFactory = DocumentBuilderFactory.newInstance();
+        Document customDocumentResult = localDocumentBuilderFactory.newDocumentBuilder().newDocument();
+
+        W3CDom w3cDom = new W3CDom();
+        w3cDom.convert(document, customDocumentResult);
+
+        String html = W3CDom.asString(customDocumentResult, W3CDom.OutputHtml());
+        assertEquals("<html><head><META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"></head><body><div></div></body></html>", html);
+    }
+
     @Test
     public void treatsUndeclaredNamespaceAsLocalName() {
         String html = "<fb:like>One</fb:like>";
diff --git a/src/test/java/org/jsoup/integration/ConnectTest.java b/src/test/java/org/jsoup/integration/ConnectTest.java
index 2956ddc8..d9f4ab3c 100644
--- a/src/test/java/org/jsoup/integration/ConnectTest.java
+++ b/src/test/java/org/jsoup/integration/ConnectTest.java
@@ -859,6 +859,19 @@ public class ConnectTest {
         assertEquals("鍵=値", URLDecoder.decode(ihVal("Query String", doc), DataUtil.UTF_8.name()));
     }
 
+    @Test void willEscapePathInRedirect() throws IOException {
+        String append = "/Foo{bar}<>%/";
+        String url = EchoServlet.Url + append;
+        Document doc = Jsoup
+            .connect(RedirectServlet.Url)
+            .data(RedirectServlet.LocationParam, url)
+            .get();
+
+        String path = ihVal("Path Info", doc);
+        assertEquals(append, path);
+        assertEquals("/EchoServlet/Foo%7Bbar%7D%3C%3E%25/", ihVal("Request URI", doc));
+    }
+
     /**
      Provides HTTP and HTTPS EchoServlet URLs
      */
diff --git a/src/test/java/org/jsoup/nodes/ElementTest.java b/src/test/java/org/jsoup/nodes/ElementTest.java
index 71a43b78..f0e1f462 100644
--- a/src/test/java/org/jsoup/nodes/ElementTest.java
+++ b/src/test/java/org/jsoup/nodes/ElementTest.java
@@ -220,6 +220,16 @@ public class ElementTest {
         assertEquals(".  ", p3.wholeText());
     }
 
+    @Test void buttonTextHasSpace() {
+        // https://github.com/jhy/jsoup/issues/2105
+        Document doc = Jsoup.parse("<html><button>Reply</button><button>All</button></html>");
+        String text = doc.body().text();
+        String wholetext = doc.body().wholeText();
+
+        assertEquals("Reply All", text);
+        assertEquals("ReplyAll", wholetext);
+    }
+
     @Test
     public void testGetSiblings() {
         Document doc = Jsoup.parse("<div><p>Hello<p id=1>there<p>this<p>is<p>an<p id=last>element</div>");
diff --git a/src/test/java/org/jsoup/parser/PositionTest.java b/src/test/java/org/jsoup/parser/PositionTest.java
index 24beec8d..49d544c5 100644
--- a/src/test/java/org/jsoup/parser/PositionTest.java
+++ b/src/test/java/org/jsoup/parser/PositionTest.java
@@ -487,6 +487,15 @@ class PositionTest {
         assertEquals("h1:0-9~12-17; id:4-6=7-8; #text:9-12; #text:17-18; h2:18-27~30-35; id:22-24=25-26; #text:27-30; h10:35-40~43-49; #text:40-43; ", track.toString());
     }
 
+    @Test void tracksFirstTextnode() {
+        // https://github.com/jhy/jsoup/issues/2106
+        String html = "foo<p></p>bar<p></p><div><b>baz</b></div>";
+        Document doc = Jsoup.parse(html, TrackingHtmlParser);
+        StringBuilder track = new StringBuilder();
+        doc.body().forEachNode(node -> accumulatePositions(node, track));
+        assertEquals("body:0-0~41-41; #text:0-3; p:3-6~6-10; #text:10-13; p:13-16~16-20; div:20-25~35-41; b:25-28~31-35; #text:28-31; ", track.toString());
+    }
+
     @Test void updateKeyMaintainsRangeLc() {
         String html = "<p xsi:CLASS=On>One</p>";
         Document doc = Jsoup.parse(html, TrackingHtmlParser);