Repository: jhy/jsoup
Pull request #1683 (2021-12-13 14:29:42)

Change insertNode and popStackToClose from private to protected
so they can be re-used in jsoup-json's JsonTreeBuilder without copy-paste
https://github.com/magicprinc/jsoup.json


Patch:
diff --git a/CHANGES b/CHANGES
index 45bffa40..5e767511 100644
--- a/CHANGES
+++ b/CHANGES
@@ -25,6 +25,13 @@ jsoup changelog
     case.
     <https://github.com/jhy/jsoup/issues/1656>
 
+  * Bugfix: when reading from SequenceInputStreams across the buffer, the input stream was closed too early, resulting
+    in missed content.
+    <https://github.com/jhy/jsoup/pull/1671>
+
+  * Bugfix: a comment with all dashes (<!----->) should not emit a parse error.
+    <https://github.com/jhy/jsoup/issues/1667>
+
 *** Release 1.14.3 [2021-Sep-30]
   * Improvement: added native XPath support in Element#selectXpath(String)
     <https://github.com/jhy/jsoup/pull/1629>
diff --git a/pom.xml b/pom.xml
index 41def937..2ab86663 100644
--- a/pom.xml
+++ b/pom.xml
@@ -136,7 +136,7 @@
       <plugin>
         <groupId>org.apache.felix</groupId>
         <artifactId>maven-bundle-plugin</artifactId>
-        <version>5.1.2</version>
+        <version>5.1.3</version>
         <executions>
           <execution>
             <id>bundle-manifest</id>
@@ -304,7 +304,7 @@
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter</artifactId>
-      <version>5.8.1</version>
+      <version>5.8.2</version>
       <scope>test</scope>
     </dependency>
 
diff --git a/src/main/java/org/jsoup/internal/ConstrainableInputStream.java b/src/main/java/org/jsoup/internal/ConstrainableInputStream.java
index c043b421..0f7e3c3b 100644
--- a/src/main/java/org/jsoup/internal/ConstrainableInputStream.java
+++ b/src/main/java/org/jsoup/internal/ConstrainableInputStream.java
@@ -81,10 +81,10 @@ public final class ConstrainableInputStream extends BufferedInputStream {
         final ByteArrayOutputStream outStream = new ByteArrayOutputStream(bufferSize);
 
         int read;
-        int remaining = max;
+        int remaining = bufferSize;
 
         while (true) {
-            read = read(readBuffer);
+            read = read(readBuffer, 0, remaining);
             if (read == -1) break;
             if (localCapped) { // this local byteBuffer cap may be smaller than the overall maxSize (like when reading first bytes)
                 if (read >= remaining) {
diff --git a/src/main/java/org/jsoup/parser/TokeniserState.java b/src/main/java/org/jsoup/parser/TokeniserState.java
index acc8db35..874fed0c 100644
--- a/src/main/java/org/jsoup/parser/TokeniserState.java
+++ b/src/main/java/org/jsoup/parser/TokeniserState.java
@@ -970,7 +970,7 @@ enum TokeniserState {
             char c = r.consume();
             switch (c) {
                 case '-':
-                    t.transition(CommentStartDash);
+                    t.transition(CommentEnd);
                     break;
                 case nullChar:
                     t.error(this);
@@ -1052,11 +1052,9 @@ enum TokeniserState {
                     t.transition(Comment);
                     break;
                 case '!':
-                    t.error(this);
                     t.transition(CommentEndBang);
                     break;
                 case '-':
-                    t.error(this);
                     t.commentPending.append('-');
                     break;
                 case eof:
@@ -1065,7 +1063,6 @@ enum TokeniserState {
                     t.transition(Data);
                     break;
                 default:
-                    t.error(this);
                     t.commentPending.append("--").append(c);
                     t.transition(Comment);
             }
diff --git a/src/main/java/org/jsoup/parser/XmlTreeBuilder.java b/src/main/java/org/jsoup/parser/XmlTreeBuilder.java
index 97ef372c..a3dc7917 100644
--- a/src/main/java/org/jsoup/parser/XmlTreeBuilder.java
+++ b/src/main/java/org/jsoup/parser/XmlTreeBuilder.java
@@ -78,7 +78,7 @@ public class XmlTreeBuilder extends TreeBuilder {
         return true;
     }
 
-    private void insertNode(Node node) {
+    protected void insertNode(Node node) {
         currentElement().appendChild(node);
     }
 
@@ -129,7 +129,7 @@ public class XmlTreeBuilder extends TreeBuilder {
      *
      * @param endTag tag to close
      */
-    private void popStackToClose(Token.EndTag endTag) {
+    protected void popStackToClose(Token.EndTag endTag) {
         // like in HtmlTreeBuilder - don't scan up forever for very (artificially) deeply nested stacks
         String elName = settings.normalizeTag(endTag.tagName);
         Element firstFound = null;


Test patch:
diff --git a/src/test/java/org/jsoup/helper/DataUtilTest.java b/src/test/java/org/jsoup/helper/DataUtilTest.java
index a775723f..59027aeb 100644
--- a/src/test/java/org/jsoup/helper/DataUtilTest.java
+++ b/src/test/java/org/jsoup/helper/DataUtilTest.java
@@ -7,6 +7,7 @@ import org.junit.jupiter.api.Test;
 
 import java.io.*;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 
 import static org.jsoup.integration.ParseTest.getFile;
 import static org.junit.jupiter.api.Assertions.*;
@@ -130,6 +131,22 @@ public class DataUtilTest {
         assertEquals("Übergrößenträger", doc.body().text());
     }
 
+    @Test
+    public void parseSequenceInputStream() throws IOException {
+        // https://github.com/jhy/jsoup/pull/1671
+        File in = getFile("/htmltests/medium.html");
+        String fileContent = new String(Files.readAllBytes(in.toPath()));
+        int halfLength = fileContent.length() / 2;
+        String firstPart = fileContent.substring(0, halfLength);
+        String secondPart = fileContent.substring(halfLength);
+        SequenceInputStream sequenceStream = new SequenceInputStream(
+                stream(firstPart),
+                stream(secondPart)
+        );
+        Document doc = DataUtil.parseInputStream(sequenceStream, null, "", Parser.htmlParser());
+        assertEquals(fileContent, doc.outerHtml());
+    }
+
     @Test
     public void supportsBOMinFiles() throws IOException {
         // test files from http://www.i18nl10n.com/korean/utftest/
diff --git a/src/test/java/org/jsoup/parser/HtmlParserTest.java b/src/test/java/org/jsoup/parser/HtmlParserTest.java
index 78d7ca32..08accbbd 100644
--- a/src/test/java/org/jsoup/parser/HtmlParserTest.java
+++ b/src/test/java/org/jsoup/parser/HtmlParserTest.java
@@ -102,6 +102,17 @@ public class HtmlParserTest {
         assertEquals(" <tr><td>", comment.getData());
     }
 
+    @Test void allDashCommentsAreNotParseErrors() {
+        // https://github.com/jhy/jsoup/issues/1667
+        // <!-----> is not a parse error
+        String html = "<!------>";
+        Parser parser = Parser.htmlParser().setTrackErrors(10);
+        Document doc = Jsoup.parse(html, parser);
+        Comment comment = (Comment) doc.childNode(0);
+        assertEquals("--", comment.getData());
+        assertEquals(0, parser.getErrors().size());
+    }
+
     @Test public void dropsUnterminatedTag() {
         // jsoup used to parse this to <p>, but whatwg, webkit will drop.
         String h1 = "<p";
diff --git a/src/test/resources/htmltests/medium.html b/src/test/resources/htmltests/medium.html
new file mode 100644
index 00000000..10f4402f
--- /dev/null
+++ b/src/test/resources/htmltests/medium.html
@@ -0,0 +1,18 @@
+<html> 
+ <head> 
+  <title>Large HTML</title> 
+ </head> 
+ <body> 
+  <p>Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer nec odio. Praesent libero. Sed cursus ante dapibus diam. Sed nisi. Nulla quis sem at nibh elementum imperdiet. Duis sagittis ipsum. Praesent mauris. Fusce nec tellus sed augue semper porta. Mauris massa. Vestibulum lacinia arcu eget nulla. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Curabitur sodales ligula in libero. </p> 
+  <p>Sed dignissim lacinia nunc. Curabitur tortor. Pellentesque nibh. Aenean quam. In scelerisque sem at dolor. Maecenas mattis. Sed convallis tristique sem. Proin ut ligula vel nunc egestas porttitor. <i>Lorem ipsum dolor sit amet, consectetur adipiscing elit</i>. Morbi lectus risus, iaculis vel, suscipit quis, luctus non, massa. Fusce ac turpis quis ligula lacinia aliquet. Mauris ipsum. <b>Aenean quam</b>. Nulla metus metus, ullamcorper vel, tincidunt sed, euismod in, nibh. </p> 
+  <p>Quisque volutpat condimentum velit. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Nam nec ante. Sed lacinia, urna non tincidunt mattis, tortor neque adipiscing diam, a cursus ipsum ante quis turpis. Nulla facilisi. Ut fringilla. Suspendisse potenti. Nunc feugiat mi a tellus consequat imperdiet. Vestibulum sapien. Proin quam. <b>Sed lacinia, urna non tincidunt mattis, tortor neque adipiscing diam, a cursus ipsum ante quis turpis</b>. Etiam ultrices. Suspendisse in justo eu magna luctus suscipit. Sed lectus. Integer euismod lacus luctus magna. </p> 
+  <p>Quisque cursus, metus vitae pharetra auctor, sem massa mattis sem, at interdum magna augue eget diam. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Morbi lacinia molestie dui. Praesent blandit dolor. Sed non quam. In vel mi sit amet augue congue elementum. Morbi in ipsum sit amet pede facilisis laoreet. <b>Suspendisse in justo eu magna luctus suscipit</b>. Donec lacus nunc, viverra nec, blandit vel, egestas et, augue. Vestibulum tincidunt malesuada tellus. Ut ultrices ultrices enim. Curabitur sit amet mauris. Morbi in dui quis est pulvinar ullamcorper. Nulla facilisi. </p> 
+  <p>Integer lacinia sollicitudin massa. <b>Quisque cursus, metus vitae pharetra auctor, sem massa mattis sem, at interdum magna augue eget diam</b>. Cras metus. Sed aliquet risus a tortor. Integer id quam. Morbi mi. <b>Morbi in dui quis est pulvinar ullamcorper</b>. Quisque nisl felis, venenatis tristique, dignissim in, ultrices sit amet, augue. Proin sodales libero eget ante. Nulla quam. Aenean laoreet. Vestibulum nisi lectus, commodo ac, facilisis ac, ultricies eu, pede. Ut orci risus, accumsan porttitor, cursus quis, aliquet eget, justo. Sed pretium blandit orci. <b>Quisque nisl felis, venenatis tristique, dignissim in, ultrices sit amet, augue</b>. Ut eu diam at pede suscipit sodales. Aenean lectus elit, fermentum non, convallis id, sagittis at, neque. </p> 
+  <p>Nullam mauris orci, aliquet et, iaculis et, viverra vitae, ligula. Nulla ut felis in purus aliquam imperdiet. Maecenas aliquet mollis lectus. Vivamus consectetuer risus et tortor. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer nec odio. Praesent libero. Sed cursus ante dapibus diam. Sed nisi. Nulla quis sem at nibh elementum imperdiet. Duis sagittis ipsum. </p> 
+  <p><i>Proin sodales libero eget ante</i>. Praesent mauris. Fusce nec tellus sed augue semper porta. <i>Nullam mauris orci, aliquet et, iaculis et, viverra vitae, ligula</i>. Mauris massa. Vestibulum lacinia arcu eget nulla. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Curabitur sodales ligula in libero. Sed dignissim lacinia nunc. Curabitur tortor. Pellentesque nibh. Aenean quam. In scelerisque sem at dolor. Maecenas mattis. </p> 
+  <p>Sed convallis tristique sem. Proin ut ligula vel nunc egestas porttitor. Morbi lectus risus, iaculis vel, suscipit quis, luctus non, massa. Fusce ac turpis quis ligula lacinia aliquet. Mauris ipsum. Nulla metus metus, ullamcorper vel, tincidunt sed, euismod in, nibh. Quisque volutpat condimentum velit. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. <b>Sed convallis tristique sem</b>. Nam nec ante. Sed lacinia, urna non tincidunt mattis, tortor neque adipiscing diam, a cursus ipsum ante quis turpis. Nulla facilisi. Ut fringilla. Suspendisse potenti. Nunc feugiat mi a tellus consequat imperdiet. </p> 
+  <p>Vestibulum sapien. Proin quam. <i>Fusce ac turpis quis ligula lacinia aliquet</i>. Etiam ultrices. Suspendisse in justo eu magna luctus suscipit. Sed lectus. Integer euismod lacus luctus magna. Quisque cursus, metus vitae pharetra auctor, sem massa mattis sem, at interdum magna augue eget diam. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Morbi lacinia molestie dui. <b>Suspendisse potenti</b>. Praesent blandit dolor. Sed non quam. In vel mi sit amet augue congue elementum. Morbi in ipsum sit amet pede facilisis laoreet. Donec lacus nunc, viverra nec, blandit vel, egestas et, augue. </p> 
+  <p>Vestibulum tincidunt malesuada tellus. Ut ultrices ultrices enim. Curabitur sit amet mauris. Morbi in dui quis est pulvinar ullamcorper. Nulla facilisi. <b>Quisque cursus, metus vitae pharetra auctor, sem massa mattis sem, at interdum magna augue eget diam</b>. Integer lacinia sollicitudin massa. Cras metus. Sed aliquet risus a tortor. Integer id quam. Morbi mi. Quisque nisl felis, venenatis tristique, dignissim in, ultrices sit amet, augue. Proin sodales libero eget ante. Nulla quam. </p> 
+  <p>Aenean laoreet. Vestibulum nisi lectus, commodo ac, facilisis ac, ultricies eu, pede. Ut orci risus, accumsan porttitor, cursus quis, aliquet eget, justo. Sed pretium blandit orci. Ut eu diam at pede suscipit sodales. Aenean lectus elit, fermentum non, convallis id, sagittis at, neque. Nullam mauris orci, aliquet et, iaculis et, viverra vitae, ligula. Nulla ut felis in purus aliquam imperdiet. Maecenas aliquet mollis lectus. Vivamus consectetuer risus et tortor. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer nec odio. </p>  
+ </body>
+</html>
\ No newline at end of file