Repository: jhy/jsoup
Pull request #1629 (2021-09-08 12:30:09)

Native XPath support in jsoup
This is a first draft of adding native XPath support to jsoup. It uses the Java provided XPath parser & evaluator, so should fully support all valid XPath expressions.

As a starting point, I've added the method:
```java
Elements el.selectXpath(String expression)
```

For example:
```java
String html = "<body><div><p>One</div><div><p>Two</div><div>Three</div>";
Document doc = Jsoup.parse(html);

Elements els = doc.selectXpath("//div/p");
assertEquals(2, els.size());
assertEquals("One", els.get(0).text());
assertEquals("Two", els.get(1).text());
```

I'm planning on bringing this in as a *beta* feature in the next release (1.14.3 -- I'll add a few more tests first) and then hopefully finalizing it in a release shortly following that. I'm looking for all feedback on how to make this feature most useful, and feel jsoup native.

Particularly: 
- does the name `selectXpath` make sense? Had thought also about just `xpath()` or `selectX`)
- should we have a `selectXpathFirst` method, like `selectFirst`?
- this current implementation is namespace aware. From looking at common XPath questions, it seems that that namespaces can be a source of confusion and query difficulty. Should we make it default to namespace unaware, and add another method (?) `selectXpathNS` or similar with it on? Or best to leave as-is?
- Does this match what you were expecting when you heard "jsoup has xpath support now"? What's missing? (Please provide suggested API signatures and how you'd use them)
- Does it work for your data and queries? Can you break it?


Patch:
diff --git a/CHANGES b/CHANGES
index 55a68004..9d1c77ba 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,9 @@
 jsoup changelog
 
 *** Release 1.14.3 [PENDING]
+  * Improvement: added native XPath support in Element#selectXpath(String)
+    <https://github.com/jhy/jsoup/pull/1629>
+
   * Improvement: added support in CharacterReader to track newlines, so that parse errors can be reported more
     intuitively.
     <https://github.com/jhy/jsoup/pull/1624>
diff --git a/pom.xml b/pom.xml
index e58b8a1d..249ed7a8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -344,7 +344,6 @@
       <version>3.0.2</version>
       <scope>provided</scope>
     </dependency>
-
   </dependencies>
 
   <dependencyManagement>
diff --git a/src/main/java/org/jsoup/Jsoup.java b/src/main/java/org/jsoup/Jsoup.java
index 1e0a3d8d..1541b03a 100644
--- a/src/main/java/org/jsoup/Jsoup.java
+++ b/src/main/java/org/jsoup/Jsoup.java
@@ -49,8 +49,22 @@ public class Jsoup {
     }
 
     /**
-     Parse HTML into a Document. As no base URI is specified, absolute URL detection relies on the HTML including a
-     {@code <base href>} tag.
+     Parse HTML into a Document, using the provided Parser. You can provide an alternate parser, such as a simple XML
+     (non-HTML) parser.  As no base URI is specified, absolute URL resolution, if required, relies on the HTML including
+     a {@code <base href>} tag.
+
+     @param html    HTML to parse
+     before the HTML declares a {@code <base href>} tag.
+     @param parser alternate {@link Parser#xmlParser() parser} to use.
+     @return sane HTML
+     */
+    public static Document parse(String html, Parser parser) {
+        return parser.parseInput(html, "");
+    }
+
+    /**
+     Parse HTML into a Document. As no base URI is specified, absolute URL resolution, if required, relies on the HTML
+     including a {@code <base href>} tag.
 
      @param html HTML to parse
      @return sane HTML
diff --git a/src/main/java/org/jsoup/helper/W3CDom.java b/src/main/java/org/jsoup/helper/W3CDom.java
index d3dae0c5..11a595df 100644
--- a/src/main/java/org/jsoup/helper/W3CDom.java
+++ b/src/main/java/org/jsoup/helper/W3CDom.java
@@ -3,8 +3,10 @@ package org.jsoup.helper;
 import org.jsoup.internal.StringUtil;
 import org.jsoup.nodes.Attribute;
 import org.jsoup.nodes.Attributes;
+import org.jsoup.select.Elements;
 import org.jsoup.select.NodeTraversor;
 import org.jsoup.select.NodeVisitor;
+import org.jsoup.select.Selector;
 import org.w3c.dom.Comment;
 import org.w3c.dom.DOMException;
 import org.w3c.dom.DOMImplementation;
@@ -12,6 +14,7 @@ import org.w3c.dom.Document;
 import org.w3c.dom.DocumentType;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
 import org.w3c.dom.Text;
 
 import javax.annotation.Nullable;
@@ -24,6 +27,11 @@ import javax.xml.transform.TransformerException;
 import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpression;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+import javax.xml.xpath.XPathFactoryConfigurationException;
 import java.io.StringWriter;
 import java.util.HashMap;
 import java.util.Map;
@@ -38,6 +46,15 @@ import static javax.xml.transform.OutputKeys.METHOD;
  * for integration with toolsets that use the W3C DOM.
  */
 public class W3CDom {
+    /** For W3C Documents created by this class, this property is set on each node to link back to the original jsoup node. */
+    public static final String SourceProperty = "jsoupSource";
+
+    /**
+     To get support for XPath versions > 1, set this property to the classname of an alternate XPathFactory
+     implementation (e.g. <code>net.sf.saxon.xpath.XPathFactoryImpl</code>).
+     */
+    public static final String XPathFactoryProperty = "javax.xml.xpath.XPathFactory:jsoup";
+
     protected DocumentBuilderFactory factory;
 
     public W3CDom() {
@@ -46,7 +63,7 @@ public class W3CDom {
     }
 
     /**
-     * Converts a jsoup DOM to a W3C DOM
+     * Converts a jsoup DOM to a W3C DOM.
      *
      * @param in jsoup Document
      * @return W3C Document
@@ -130,12 +147,27 @@ public class W3CDom {
     }
 
     /**
-     * Convert a jsoup Document to a W3C Document.
+     * Convert a jsoup Document to a W3C Document. The created nodes will link back to the original
+     * jsoup nodes in the user property {@link #SourceProperty} (but after conversion, changes on one side will not
+     * flow to the other).
      *
      * @param in jsoup doc
-     * @return w3c doc
+     * @return a W3C DOM Document representing the jsoup Document or Element contents.
      */
     public Document fromJsoup(org.jsoup.nodes.Document in) {
+        // just method API backcompat
+        return fromJsoup((org.jsoup.nodes.Element) in);
+    }
+
+    /**
+     * Convert a jsoup Element to a W3C Document. The created nodes will link back to the original
+     * jsoup nodes in the user property {@link #SourceProperty} (but after conversion, changes on one side will not
+     * flow to the other).
+     *
+     * @param in jsoup element or doc
+     * @return a W3C DOM Document representing the jsoup Document or Element contents.
+     */
+    public Document fromJsoup(org.jsoup.nodes.Element in) {
         Validate.notNull(in);
         DocumentBuilder builder;
         try {
@@ -144,7 +176,8 @@ public class W3CDom {
             Document out;
 
             out = builder.newDocument();
-            org.jsoup.nodes.DocumentType doctype = in.documentType();
+            org.jsoup.nodes.Document inDoc = in.ownerDocument();
+            org.jsoup.nodes.DocumentType doctype = inDoc != null ? inDoc.documentType() : null;
             if (doctype != null) {
                 org.w3c.dom.DocumentType documentType = impl.createDocumentType(doctype.name(), doctype.publicId(), doctype.systemId());
                 out.appendChild(documentType);
@@ -159,21 +192,70 @@ public class W3CDom {
     }
 
     /**
-     * Converts a jsoup document into the provided W3C Document. If required, you can set options on the output document
-     * before converting.
+     * Converts a jsoup document into the provided W3C Document. If required, you can set options on the output
+     * document before converting.
      *
      * @param in jsoup doc
      * @param out w3c doc
-     * @see org.jsoup.helper.W3CDom#fromJsoup(org.jsoup.nodes.Document)
+     * @see org.jsoup.helper.W3CDom#fromJsoup(org.jsoup.nodes.Element)
      */
     public void convert(org.jsoup.nodes.Document in, Document out) {
-        if (!StringUtil.isBlank(in.location()))
-            out.setDocumentURI(in.location());
+        // just provides method API backcompat
+        convert((org.jsoup.nodes.Element) in, out);
+    }
 
-        org.jsoup.nodes.Element rootEl = in.child(0); // skip the #root node
+    /**
+     * Converts a jsoup element into the provided W3C Document. If required, you can set options on the output
+     * document before converting.
+     *
+     * @param in jsoup element
+     * @param out w3c doc
+     * @see org.jsoup.helper.W3CDom#fromJsoup(org.jsoup.nodes.Element)
+     */
+    public void convert(org.jsoup.nodes.Element in, Document out) {
+        org.jsoup.nodes.Document inDoc = in.ownerDocument();
+        if (inDoc != null) {
+            if (!StringUtil.isBlank(inDoc.location()))
+                out.setDocumentURI(inDoc.location());
+        }
+
+        org.jsoup.nodes.Element rootEl = in instanceof org.jsoup.nodes.Document ? in.child(0) : in; // skip the #root node if a Document
         NodeTraversor.traverse(new W3CBuilder(out), rootEl);
     }
 
+    public NodeList selectXpath(String xpath, Document doc) {
+        Validate.notEmpty(xpath);
+        Validate.notNull(doc);
+
+        NodeList nodeList;
+        try {
+            // if there is a configured XPath factory, use that instead of the Java base impl:
+            String property = System.getProperty(XPathFactoryProperty);
+            final XPathFactory xPathFactory = property != null ?
+                XPathFactory.newInstance("jsoup") :
+                XPathFactory.newInstance();
+
+            XPathExpression expression = xPathFactory.newXPath().compile(xpath);
+            nodeList = (NodeList) expression.evaluate(doc, XPathConstants.NODESET); // love the strong typing here /s
+            Validate.notNull(nodeList);
+        } catch (XPathExpressionException | XPathFactoryConfigurationException e) {
+            throw new Selector.SelectorParseException("Could not evaluate XPath query [%s]: %s", xpath, e.getMessage());
+        }
+        return nodeList;
+    }
+
+    public Elements sourceElements(NodeList nodeList) {
+        Elements els = new Elements();
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            org.w3c.dom.Node node = nodeList.item(i);
+            Object source = node.getUserData(W3CDom.SourceProperty);
+            if (source instanceof org.jsoup.nodes.Element)
+                els.add((org.jsoup.nodes.Element) source);
+        }
+
+        return els;
+    }
+
     /**
      * Serialize a W3C document to a String. The output format will be XML or HTML depending on the content of the doc.
      *
@@ -211,7 +293,7 @@ public class W3CDom {
                 String namespace = namespacesStack.peek().get(prefix);
                 String tagName = sourceEl.tagName();
 
-                /* Tag names in XML are quite, but less, permissive than HTML. Rather than reimplement the validation,
+                /* Tag names in XML are quite permissive, but less permissive than HTML. Rather than reimplement the validation,
                 we just try to use it as-is. If it fails, insert as a text node instead. We don't try to normalize the
                 tagname to something safe, because that isn't going to be meaningful downstream. This seems(?) to be
                 how browsers handle the situation, also. https://github.com/jhy/jsoup/issues/1093 */
@@ -220,32 +302,36 @@ public class W3CDom {
                         doc.createElementNS("", tagName) : // doesn't have a real namespace defined
                         doc.createElementNS(namespace, tagName);
                     copyAttributes(sourceEl, el);
-                    dest.appendChild(el);
+                    append(el, sourceEl);
                     dest = el; // descend
                 } catch (DOMException e) {
-                    dest.appendChild(doc.createTextNode("<" + tagName + ">"));
+                    append(doc.createTextNode("<" + tagName + ">"), sourceEl);
                 }
             } else if (source instanceof org.jsoup.nodes.TextNode) {
                 org.jsoup.nodes.TextNode sourceText = (org.jsoup.nodes.TextNode) source;
                 Text text = doc.createTextNode(sourceText.getWholeText());
-                dest.appendChild(text);
+                append(text, sourceText);
             } else if (source instanceof org.jsoup.nodes.Comment) {
                 org.jsoup.nodes.Comment sourceComment = (org.jsoup.nodes.Comment) source;
                 Comment comment = doc.createComment(sourceComment.getData());
-                dest.appendChild(comment);
+                append(comment, sourceComment);
             } else if (source instanceof org.jsoup.nodes.DataNode) {
                 org.jsoup.nodes.DataNode sourceData = (org.jsoup.nodes.DataNode) source;
                 Text node = doc.createTextNode(sourceData.getWholeData());
-                dest.appendChild(node);
+                append(node, sourceData);
             } else {
-                // unhandled
-                // not that doctype is not handled here - rather it is used in the initial doc creation
+                // unhandled. note that doctype is not handled here - rather it is used in the initial doc creation
             }
         }
 
+        private void append(Node append, org.jsoup.nodes.Node source) {
+            append.setUserData(SourceProperty, source, null);
+            dest.appendChild(append);
+        }
+
         public void tail(org.jsoup.nodes.Node source, int depth) {
             if (source instanceof org.jsoup.nodes.Element && dest.getParentNode() instanceof Element) {
-                dest = dest.getParentNode(); // undescend. cromulent.
+                dest = dest.getParentNode(); // undescend
             }
             namespacesStack.pop();
         }
diff --git a/src/main/java/org/jsoup/nodes/Element.java b/src/main/java/org/jsoup/nodes/Element.java
index 0a0180bc..c1e08190 100644
--- a/src/main/java/org/jsoup/nodes/Element.java
+++ b/src/main/java/org/jsoup/nodes/Element.java
@@ -34,9 +34,9 @@ import static org.jsoup.internal.Normalizer.normalize;
 /**
  * A HTML element consists of a tag name, attributes, and child nodes (including text nodes and
  * other elements).
- * 
+ *
  * From an Element, you can extract data, traverse the node graph, and manipulate the HTML.
- * 
+ *
  * @author Jonathan Hedley, jonathan@hedley.net
  */
 @NonnullByDefault
@@ -59,7 +59,7 @@ public class Element extends Node {
 
     /**
      * Create a new, standalone Element. (Standalone in that is has no parent.)
-     * 
+     *
      * @param tag tag of this element
      * @param baseUri the base URI (optional, may be null to inherit from parent, or "" to clear parent's)
      * @param attributes initial attributes (optional, may be null)
@@ -77,7 +77,7 @@ public class Element extends Node {
 
     /**
      * Create a new Element from a Tag and a base URI.
-     * 
+     *
      * @param tag element tag
      * @param baseUri the base URI of this element. Optional, and will inherit from its parent, if any.
      * @see Tag#valueOf(String, ParseSettings)
@@ -145,7 +145,7 @@ public class Element extends Node {
     /**
      * Get the name of the tag for this element. E.g. {@code div}. If you are using {@link ParseSettings#preserveCase
      * case preserving parsing}, this will return the source's original case.
-     * 
+     *
      * @return the tag name
      */
     public String tagName() {
@@ -168,7 +168,7 @@ public class Element extends Node {
      *
      * @param tagName new tag name for this element
      * @return this element, for chaining
-     * @see Elements#tagName(String) 
+     * @see Elements#tagName(String)
      */
     public Element tagName(String tagName) {
         Validate.notEmpty(tagName, "Tag name must not be empty.");
@@ -178,17 +178,17 @@ public class Element extends Node {
 
     /**
      * Get the Tag for this element.
-     * 
+     *
      * @return the tag object
      */
     public Tag tag() {
         return tag;
     }
-    
+
     /**
      * Test if this element is a block-level element. (E.g. {@code <div> == true} or an inline element
      * {@code <span> == false}).
-     * 
+     *
      * @return true if block, false if not (and thus inline)
      */
     public boolean isBlock() {
@@ -197,7 +197,7 @@ public class Element extends Node {
 
     /**
      * Get the {@code id} attribute of this element.
-     * 
+     *
      * @return The id attribute, if present, or an empty string if not.
      */
     public String id() {
@@ -218,22 +218,22 @@ public class Element extends Node {
     /**
      * Set an attribute value on this element. If this element already has an attribute with the
      * key, its value is updated; otherwise, a new attribute is added.
-     * 
+     *
      * @return this element
      */
     public Element attr(String attributeKey, String attributeValue) {
         super.attr(attributeKey, attributeValue);
         return this;
     }
-    
+
     /**
      * Set a boolean attribute value on this element. Setting to <code>true</code> sets the attribute value to "" and
      * marks the attribute as boolean so no value is written out. Setting to <code>false</code> removes the attribute
      * with the same key if it exists.
-     * 
+     *
      * @param attributeKey the attribute key
      * @param attributeValue the attribute value
-     * 
+     *
      * @return this element
      */
     public Element attr(String attributeKey, boolean attributeValue) {
@@ -287,7 +287,7 @@ public class Element extends Node {
      * Note that an element can have both mixed Nodes and Elements as children. This method inspects
      * a filtered list of children that are elements, and the index is based on that filtered list.
      * </p>
-     * 
+     *
      * @param index the index number of the element to retrieve
      * @return the child element, if it exists, otherwise throws an {@code IndexOutOfBoundsException}
      * @see #childNode(int)
@@ -410,7 +410,7 @@ public class Element extends Node {
      * </ul>
      * <p>See the query syntax documentation in {@link org.jsoup.select.Selector}.</p>
      * <p>Also known as {@code querySelectorAll()} in the Web DOM.</p>
-     * 
+     *
      * @param cssQuery a {@link Selector} CSS-like query
      * @return an {@link Elements} list containing elements that match the query (empty if none match)
      * @see Selector selector query syntax
@@ -506,10 +506,30 @@ public class Element extends Node {
         } while (el != null);
         return null;
     }
-    
+
+    /**
+     <b>Beta:</b> find Elements that match the supplied XPath expression.
+     <p>(This functionality is currently in beta and
+     is subject to change. Feedback on the API is requested and welcomed!)</p>
+     <p>By default, XPath 1.0 expressions are supported. If you would to use XPath 2.0 or higher, you can provide an
+     alternate XPathFactory implementation:</p>
+     <ol>
+        <li>Add the implementation to your classpath. E.g. to use <a href="https://www.saxonica.com/products/products.xml">Saxon-HE</a>, add <a href="https://mvnrepository.com/artifact/net.sf.saxon/Saxon-HE">net.sf.saxon:Saxon-HE</a> to your build.</li>
+     <li>Set the system property <code>javax.xml.xpath.XPathFactory:jsoup</code> to the implementing classname. E.g.:<br>
+        <code>System.setProperty(W3CDom.XPathFactoryProperty, "net.sf.saxon.xpath.XPathFactoryImpl");</code>
+     </li>
+     </ol>
+
+     @param xpath XML path expression
+     @return matching elements, or an empty list if none match.
+     */
+    public Elements selectXpath(String xpath) {
+        return NodeUtils.selectXpath(xpath, this);
+    }
+
     /**
      * Insert a node to the end of this Element's children. The incoming node will be re-parented.
-     * 
+     *
      * @param child node to add.
      * @return this Element, for chaining
      * @see #prependChild(Node)
@@ -552,13 +572,13 @@ public class Element extends Node {
 
     /**
      * Add a node to the start of this element's children.
-     * 
+     *
      * @param child node to add.
      * @return this element, so that you can add more child nodes or elements.
      */
     public Element prependChild(Node child) {
         Validate.notNull(child);
-        
+
         addChildren(0, child);
         return this;
     }
@@ -615,10 +635,10 @@ public class Element extends Node {
         addChildren(index, children);
         return this;
     }
-    
+
     /**
      * Create a new element by tag name, and add it as the last child.
-     * 
+     *
      * @param tagName the name of the tag (e.g. {@code div}).
      * @return the new element, to allow you to add content to it, e.g.:
      *  {@code parent.appendElement("h1").attr("id", "header").text("Welcome");}
@@ -628,10 +648,10 @@ public class Element extends Node {
         appendChild(child);
         return child;
     }
-    
+
     /**
      * Create a new element by tag name, and add it as the first child.
-     * 
+     *
      * @param tagName the name of the tag (e.g. {@code div}).
      * @return the new element, to allow you to add content to it, e.g.:
      *  {@code parent.prependElement("h1").attr("id", "header").text("Welcome");}
@@ -641,10 +661,10 @@ public class Element extends Node {
         prependChild(child);
         return child;
     }
-    
+
     /**
      * Create and append a new TextNode to this element.
-     * 
+     *
      * @param text the unencoded text to add
      * @return this element
      */
@@ -654,10 +674,10 @@ public class Element extends Node {
         appendChild(node);
         return this;
     }
-    
+
     /**
      * Create and prepend a new TextNode to this element.
-     * 
+     *
      * @param text the unencoded text to add
      * @return this element
      */
@@ -667,7 +687,7 @@ public class Element extends Node {
         prependChild(node);
         return this;
     }
-    
+
     /**
      * Add inner HTML to this element. The supplied HTML will be parsed, and each node appended to the end of the children.
      * @param html HTML to add inside this element, after the existing HTML
@@ -680,7 +700,7 @@ public class Element extends Node {
         addChildren(nodes.toArray(new Node[0]));
         return this;
     }
-    
+
     /**
      * Add inner HTML into this element. The supplied HTML will be parsed, and each node prepended to the start of the element's children.
      * @param html HTML to add inside this element, before the existing HTML
@@ -821,7 +841,7 @@ public class Element extends Node {
     }
 
     /**
-     * Gets the next sibling element of this element. E.g., if a {@code div} contains two {@code p}s, 
+     * Gets the next sibling element of this element. E.g., if a {@code div} contains two {@code p}s,
      * the {@code nextElementSibling} of the first {@code p} is the second {@code p}.
      * <p>
      * This is similar to {@link #nextSibling()}, but specifically finds only Elements
@@ -882,7 +902,7 @@ public class Element extends Node {
 
     /**
      * Gets the first Element sibling of this element. That may be this element.
-     * @return the first sibling that is an element (aka the parent's first element child) 
+     * @return the first sibling that is an element (aka the parent's first element child)
      */
     public Element firstElementSibling() {
         if (parent() != null) {
@@ -891,7 +911,7 @@ public class Element extends Node {
         } else
             return this; // orphan is its own first sibling
     }
-    
+
     /**
      * Get the list index of this element in its element sibling list. I.e. if this is the first element
      * sibling, returns 0.
@@ -904,7 +924,7 @@ public class Element extends Node {
 
     /**
      * Gets the last element sibling of this element. That may be this element.
-     * @return the last sibling that is an element (aka the parent's last element child) 
+     * @return the last sibling that is an element (aka the parent's last element child)
      */
     public Element lastElementSibling() {
         if (parent() != null) {
@@ -948,7 +968,7 @@ public class Element extends Node {
      */
     public @Nullable Element getElementById(String id) {
         Validate.notEmpty(id);
-        
+
         Elements elements = Collector.collect(new Evaluator.Id(id), this);
         if (elements.size() > 0)
             return elements.get(0);
@@ -961,7 +981,7 @@ public class Element extends Node {
      * <p>
      * Elements can have multiple classes (e.g. {@code <div class="header round first">}. This method
      * checks each class, so you can find the above with {@code el.getElementsByClass("header");}.
-     * 
+     *
      * @param className the name of the class to search for.
      * @return elements with the supplied class name, empty if none
      * @see #hasClass(String)
@@ -1001,7 +1021,7 @@ public class Element extends Node {
 
     /**
      * Find elements that have an attribute with the specific value. Case insensitive.
-     * 
+     *
      * @param key name of the attribute
      * @param value value of the attribute
      * @return elements that have this attribute with this value, empty if none
@@ -1012,7 +1032,7 @@ public class Element extends Node {
 
     /**
      * Find elements that either do not have this attribute, or have it with a different value. Case insensitive.
-     * 
+     *
      * @param key name of the attribute
      * @param value value of the attribute
      * @return elements that do not have a matching attribute
@@ -1023,7 +1043,7 @@ public class Element extends Node {
 
     /**
      * Find elements that have attributes that start with the value prefix. Case insensitive.
-     * 
+     *
      * @param key name of the attribute
      * @param valuePrefix start of attribute value
      * @return elements that have attributes that start with the value prefix
@@ -1034,7 +1054,7 @@ public class Element extends Node {
 
     /**
      * Find elements that have attributes that end with the value suffix. Case insensitive.
-     * 
+     *
      * @param key name of the attribute
      * @param valueSuffix end of the attribute value
      * @return elements that have attributes that end with the value suffix
@@ -1045,7 +1065,7 @@ public class Element extends Node {
 
     /**
      * Find elements that have attributes whose value contains the match string. Case insensitive.
-     * 
+     *
      * @param key name of the attribute
      * @param match substring of value to search for
      * @return elements that have attributes containing this text
@@ -1053,7 +1073,7 @@ public class Element extends Node {
     public Elements getElementsByAttributeValueContaining(String key, String match) {
         return Collector.collect(new Evaluator.AttributeWithValueContaining(key, match), this);
     }
-    
+
     /**
      * Find elements that have attributes whose values match the supplied regular expression.
      * @param key name of the attribute
@@ -1062,9 +1082,9 @@ public class Element extends Node {
      */
     public Elements getElementsByAttributeValueMatching(String key, Pattern pattern) {
         return Collector.collect(new Evaluator.AttributeWithValueMatching(key, pattern), this);
-        
+
     }
-    
+
     /**
      * Find elements that have attributes whose values match the supplied regular expression.
      * @param key name of the attribute
@@ -1080,7 +1100,7 @@ public class Element extends Node {
         }
         return getElementsByAttributeValueMatching(key, pattern);
     }
-    
+
     /**
      * Find elements whose sibling index is less than the supplied index.
      * @param index 0-based index
@@ -1089,7 +1109,7 @@ public class Element extends Node {
     public Elements getElementsByIndexLessThan(int index) {
         return Collector.collect(new Evaluator.IndexLessThan(index), this);
     }
-    
+
     /**
      * Find elements whose sibling index is greater than the supplied index.
      * @param index 0-based index
@@ -1098,7 +1118,7 @@ public class Element extends Node {
     public Elements getElementsByIndexGreaterThan(int index) {
         return Collector.collect(new Evaluator.IndexGreaterThan(index), this);
     }
-    
+
     /**
      * Find elements whose sibling index is equal to the supplied index.
      * @param index 0-based index
@@ -1107,7 +1127,7 @@ public class Element extends Node {
     public Elements getElementsByIndexEquals(int index) {
         return Collector.collect(new Evaluator.IndexEquals(index), this);
     }
-    
+
     /**
      * Find elements that contain the specified string. The search is case insensitive. The text may appear directly
      * in the element, or in any of its descendants.
@@ -1118,7 +1138,7 @@ public class Element extends Node {
     public Elements getElementsContainingText(String searchText) {
         return Collector.collect(new Evaluator.ContainsText(searchText), this);
     }
-    
+
     /**
      * Find elements that directly contain the specified string. The search is case insensitive. The text must appear directly
      * in the element, not in any of its descendants.
@@ -1129,7 +1149,7 @@ public class Element extends Node {
     public Elements getElementsContainingOwnText(String searchText) {
         return Collector.collect(new Evaluator.ContainsOwnText(searchText), this);
     }
-    
+
     /**
      * Find elements whose text matches the supplied regular expression.
      * @param pattern regular expression to match text against
@@ -1139,7 +1159,7 @@ public class Element extends Node {
     public Elements getElementsMatchingText(Pattern pattern) {
         return Collector.collect(new Evaluator.Matches(pattern), this);
     }
-    
+
     /**
      * Find elements whose text matches the supplied regular expression.
      * @param regex regular expression to match text against. You can use <a href="http://java.sun.com/docs/books/tutorial/essential/regex/pattern.html#embedded">embedded flags</a> (such as (?i) and (?m) to control regex options.
@@ -1155,7 +1175,7 @@ public class Element extends Node {
         }
         return getElementsMatchingText(pattern);
     }
-    
+
     /**
      * Find elements whose own text matches the supplied regular expression.
      * @param pattern regular expression to match text against
@@ -1165,7 +1185,7 @@ public class Element extends Node {
     public Elements getElementsMatchingOwnText(Pattern pattern) {
         return Collector.collect(new Evaluator.MatchesOwn(pattern), this);
     }
-    
+
     /**
      * Find elements whose own text matches the supplied regular expression.
      * @param regex regular expression to match text against. You can use <a href="http://java.sun.com/docs/books/tutorial/essential/regex/pattern.html#embedded">embedded flags</a> (such as (?i) and (?m) to control regex options.
@@ -1181,10 +1201,10 @@ public class Element extends Node {
         }
         return getElementsMatchingOwnText(pattern);
     }
-    
+
     /**
      * Find all elements under this element (including self, and children of children).
-     * 
+     *
      * @return all elements
      */
     public Elements getAllElements() {
@@ -1389,7 +1409,7 @@ public class Element extends Node {
             }
         }
         return StringUtil.releaseBuilder(sb);
-    }   
+    }
 
     /**
      * Gets the literal value of this element's "class" attribute, which may include multiple class names, space
@@ -1528,7 +1548,7 @@ public class Element extends Node {
 
         return this;
     }
-    
+
     /**
      * Get the value of a form element (input, textarea, etc).
      * @return the value of the form element, or empty string if not set.
@@ -1539,7 +1559,7 @@ public class Element extends Node {
         else
             return attr("value");
     }
-    
+
     /**
      * Set the value of a form element (input, textarea, etc).
      * @param value value to set
@@ -1589,7 +1609,7 @@ public class Element extends Node {
     /**
      * Retrieves the element's inner HTML. E.g. on a {@code <div>} with one empty {@code <p>}, would return
      * {@code <p></p>}. (Whereas {@link #outerHtml()} would return {@code <div><p></p></div>}.)
-     * 
+     *
      * @return String of HTML.
      * @see #outerHtml()
      */
@@ -1608,7 +1628,7 @@ public class Element extends Node {
 
         return appendable;
     }
-    
+
     /**
      * Set this element's inner HTML. Clears the existing HTML first.
      * @param html HTML to parse and set into this element
diff --git a/src/main/java/org/jsoup/nodes/NodeUtils.java b/src/main/java/org/jsoup/nodes/NodeUtils.java
index 2a398390..107f4a8b 100644
--- a/src/main/java/org/jsoup/nodes/NodeUtils.java
+++ b/src/main/java/org/jsoup/nodes/NodeUtils.java
@@ -1,7 +1,11 @@
 package org.jsoup.nodes;
 
+import org.jsoup.helper.Validate;
+import org.jsoup.helper.W3CDom;
 import org.jsoup.parser.HtmlTreeBuilder;
 import org.jsoup.parser.Parser;
+import org.jsoup.select.Elements;
+import org.w3c.dom.NodeList;
 
 /**
  * Internal helpers for Nodes, to keep the actual node APIs relatively clean. A jsoup internal class, so don't use it as
@@ -24,4 +28,20 @@ final class NodeUtils {
         Document doc = node.ownerDocument();
         return doc != null && doc.parser() != null ? doc.parser() : new Parser(new HtmlTreeBuilder());
     }
+
+    /**
+     This impl works by compiling the input xpath expression, and then evaluating it against a W3C Document converted
+     from the original jsoup element. The original jsoup elements are then fetched from the w3c doc user data (where we
+     stashed them during conversion). This process could potentially be optimized by transpiling the compiled xpath
+     expression to a jsoup Evaluator when there's 1:1 support, thus saving the W3C document conversion stage.
+     */
+    static Elements selectXpath(String xpath, Element el) {
+        Validate.notEmpty(xpath);
+        Validate.notNull(el);
+
+        W3CDom w3c = new W3CDom();
+        org.w3c.dom.Document wDoc = w3c.fromJsoup(el);
+        NodeList nodeList = w3c.selectXpath(xpath, wDoc);
+        return w3c.sourceElements(nodeList);
+    }
 }


Test patch:
diff --git a/src/test/java/org/jsoup/TextUtil.java b/src/test/java/org/jsoup/TextUtil.java
index ca49bef8..54bf5b7a 100644
--- a/src/test/java/org/jsoup/TextUtil.java
+++ b/src/test/java/org/jsoup/TextUtil.java
@@ -8,12 +8,23 @@ import java.util.regex.Pattern;
  @author Jonathan Hedley, jonathan@hedley.net */
 public class TextUtil {
     static Pattern stripper = Pattern.compile("\\r?\\n\\s*");
+    static Pattern stripLines = Pattern.compile("\\r?\\n?");
+    static Pattern spaceCollapse = Pattern.compile("\\s{2,}");
+    static Pattern tagSpaceCollapse = Pattern.compile(">\\s+<");
     static Pattern stripCRs = Pattern.compile("\\r*");
 
     public static String stripNewlines(String text) {
         return stripper.matcher(text).replaceAll("");
     }
 
+    public static String normalizeSpaces(String text) {
+        text = stripLines.matcher(text).replaceAll("");
+        text = stripper.matcher(text).replaceAll("");
+        text = spaceCollapse.matcher(text).replaceAll(" ");
+        text = tagSpaceCollapse.matcher(text).replaceAll("><");
+        return text;
+    }
+
     public static String stripCRs(String text) {
         return stripCRs.matcher(text).replaceAll("");
     }
diff --git a/src/test/java/org/jsoup/helper/W3CDomTest.java b/src/test/java/org/jsoup/helper/W3CDomTest.java
index c2253bf5..1c172a4d 100644
--- a/src/test/java/org/jsoup/helper/W3CDomTest.java
+++ b/src/test/java/org/jsoup/helper/W3CDomTest.java
@@ -4,6 +4,7 @@ import org.jsoup.Jsoup;
 import org.jsoup.TextUtil;
 import org.jsoup.integration.ParseTest;
 import org.jsoup.nodes.Element;
+import org.jsoup.nodes.TextNode;
 import org.junit.jupiter.api.Test;
 import org.w3c.dom.Document;
 import org.w3c.dom.DocumentType;
@@ -23,6 +24,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
+import java.util.Locale;
 import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.*;
@@ -259,27 +261,28 @@ public class W3CDomTest {
     @Test
     public void testRoundTripDoctype() {
         // TODO - not super happy with this output - but plain DOM doesn't let it out, and don't want to rebuild the writer
+        // because we have Saxon on the test classpath, the transformer will change to that, and so case may change (e.g. Java base in META, Saxon is meta for HTML)
         String base = "<!DOCTYPE html><p>One</p>";
-        assertEquals("<!DOCTYPE html SYSTEM \"about:legacy-compat\"><html><head><META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"></head><body><p>One</p></body></html>",
+        assertEqualsIgnoreCase("<!DOCTYPE html SYSTEM \"about:legacy-compat\"><html><head><META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"></head><body><p>One</p></body></html>",
             output(base, true));
-        assertEquals("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE html SYSTEM \"about:legacy-compat\"><html><head/><body><p>One</p></body></html>", output(base, false));
+        assertEqualsIgnoreCase("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE html SYSTEM \"about:legacy-compat\"><html><head/><body><p>One</p></body></html>", output(base, false));
 
         String publicDoc = "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">";
-        assertEquals("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\"><html><head><META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"></head><body></body></html>", output(publicDoc, true));
+        assertEqualsIgnoreCase("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\"><html><head><META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"></head><body></body></html>", output(publicDoc, true));
         // different impls will have different XML formatting. OpenJDK 13 default gives this: <body /> but others have <body/>, so just check start
         assertTrue(output(publicDoc, false).startsWith("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE html PUBLIC"));
 
         String systemDoc = "<!DOCTYPE html SYSTEM \"exampledtdfile.dtd\">";
-        assertEquals("<!DOCTYPE html SYSTEM \"exampledtdfile.dtd\"><html><head><META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"></head><body></body></html>", output(systemDoc, true));
-        assertEquals("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE html SYSTEM \"exampledtdfile.dtd\"><html><head/><body/></html>", output(systemDoc, false));
+        assertEqualsIgnoreCase("<!DOCTYPE html SYSTEM \"exampledtdfile.dtd\"><html><head><META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"></head><body></body></html>", output(systemDoc, true));
+        assertEqualsIgnoreCase("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE html SYSTEM \"exampledtdfile.dtd\"><html><head/><body/></html>", output(systemDoc, false));
 
         String legacyDoc = "<!DOCTYPE html SYSTEM \"about:legacy-compat\">";
-        assertEquals("<!DOCTYPE html SYSTEM \"about:legacy-compat\"><html><head><META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"></head><body></body></html>", output(legacyDoc, true));
-        assertEquals("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE html SYSTEM \"about:legacy-compat\"><html><head/><body/></html>", output(legacyDoc, false));
+        assertEqualsIgnoreCase("<!DOCTYPE html SYSTEM \"about:legacy-compat\"><html><head><META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"></head><body></body></html>", output(legacyDoc, true));
+        assertEqualsIgnoreCase("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE html SYSTEM \"about:legacy-compat\"><html><head/><body/></html>", output(legacyDoc, false));
 
         String noDoctype = "<p>One</p>";
-        assertEquals("<html><head><META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"></head><body><p>One</p></body></html>", output(noDoctype, true));
-        assertEquals("<?xml version=\"1.0\" encoding=\"UTF-8\"?><html><head/><body><p>One</p></body></html>", output(noDoctype, false));
+        assertEqualsIgnoreCase("<html><head><META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"></head><body><p>One</p></body></html>", output(noDoctype, true));
+        assertEqualsIgnoreCase("<?xml version=\"1.0\" encoding=\"UTF-8\"?><html><head/><body><p>One</p></body></html>", output(noDoctype, false));
     }
 
     private String output(String in, boolean modeHtml) {
@@ -287,7 +290,30 @@ public class W3CDomTest {
         Document w3c = W3CDom.convert(jdoc);
 
         Map<String, String> properties = modeHtml ? W3CDom.OutputHtml() : W3CDom.OutputXml();
-        return TextUtil.stripNewlines(W3CDom.asString(w3c, properties));
+        return TextUtil.normalizeSpaces(W3CDom.asString(w3c, properties));
+    }
+
+    private void assertEqualsIgnoreCase(String want, String have) {
+        assertEquals(want.toLowerCase(Locale.ROOT), have.toLowerCase(Locale.ROOT));
+    }
+
+    @Test public void convertsElementsAndMaintainsSource() {
+        org.jsoup.nodes.Document jdoc = Jsoup.parse("<body><div><p>One</div><div><p>Two");
+        W3CDom w3CDom = new W3CDom();
+        Element jDiv = jdoc.selectFirst("div");
+        assertNotNull(jDiv);
+        Document doc = w3CDom.fromJsoup(jDiv);
+        Node div = doc.getFirstChild();
+
+        assertEquals("div", div.getLocalName());
+        assertEquals(jDiv, div.getUserData(W3CDom.SourceProperty));
+
+        Node textNode = div.getFirstChild().getFirstChild();
+        assertEquals("One", textNode.getTextContent());
+        assertEquals(Node.TEXT_NODE, textNode.getNodeType());
+
+        org.jsoup.nodes.TextNode jText = (TextNode) jDiv.childNode(0).childNode(0);
+        assertEquals(jText, textNode.getUserData(W3CDom.SourceProperty));
     }
 
 }
diff --git a/src/test/java/org/jsoup/select/XpathTest.java b/src/test/java/org/jsoup/select/XpathTest.java
new file mode 100644
index 00000000..e207be56
--- /dev/null
+++ b/src/test/java/org/jsoup/select/XpathTest.java
@@ -0,0 +1,177 @@
+package org.jsoup.select;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.parser.Parser;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathFactory;
+import javax.xml.xpath.XPathFactoryConfigurationException;
+import javax.xml.xpath.XPathFunctionResolver;
+import javax.xml.xpath.XPathVariableResolver;
+
+import java.util.stream.Stream;
+
+import static org.jsoup.helper.W3CDom.XPathFactoryProperty;
+import static org.junit.jupiter.api.Assertions.*;
+
+public class XpathTest {
+
+    @Test
+    public void supportsXpath() {
+        String html = "<body><div><p>One</div><div><p>Two</div><div>Three</div>";
+        Document doc = Jsoup.parse(html);
+
+        Elements els = doc.selectXpath("//div/p");
+        assertEquals(2, els.size());
+        assertEquals("One", els.get(0).text());
+        assertEquals("Two", els.get(1).text());
+    }
+
+    @Test public void supportsXpathFromElement() {
+        String html = "<body><div><p>One</div><div><p>Two</div><div>Three</div>";
+        Document doc = Jsoup.parse(html);
+
+        Element div = doc.selectFirst("div");
+        assertNotNull(div);
+
+        Elements els = div.selectXpath("/div/p");
+        assertEquals(1, els.size());
+        assertEquals("One", els.get(0).text());
+        assertEquals("p", els.get(0).tagName());
+
+        assertEquals(0, div.selectXpath("//body").size());
+        assertEquals(1, doc.selectXpath("//body").size());
+    }
+
+    @Test public void emptyElementsIfNoResults() {
+        Document doc = Jsoup.parse("<p>One<p>Two");
+        assertEquals(0, doc.selectXpath("//div").size());
+    }
+
+    @Test
+    public void throwsSelectException() {
+        Document doc = Jsoup.parse("<p>One<p>Two");
+        boolean threw = false;
+        try {
+            doc.selectXpath("//???");
+        } catch (Selector.SelectorParseException e) {
+            threw = true;
+            // checks exception message within jsoup's control, rest may be JDK impl specific
+            // was - Could not evaluate XPath query [//???]: javax.xml.transform.TransformerException: A location step was expected following the '/' or '//' token.
+            assertTrue(e.getMessage().startsWith("Could not evaluate XPath query [//???]:"));
+        }
+        assertTrue(threw);
+    }
+
+    @Test
+    public void supportsNamespaces() {
+        String xhtml = "<html xmlns='http://www.w3.org/1999/xhtml'><body id='One'><div>hello</div></body></html>";;
+        Document doc = Jsoup.parse(xhtml, Parser.xmlParser());
+        Elements elements = doc.selectXpath("//*[local-name()='body']");
+        assertEquals(1, elements.size());
+        assertEquals("One", elements.first().id());
+    }
+
+    @Test
+    public void canDitchNamespaces() {
+        String xhtml = "<html xmlns='http://www.w3.org/1999/xhtml'><body id='One'><div>hello</div></body></html>";;
+        Document doc = Jsoup.parse(xhtml, Parser.xmlParser());
+        doc.select("[xmlns]").removeAttr("xmlns");
+        Elements elements = doc.selectXpath("//*[local-name()='body']");
+        assertEquals(1, elements.size());
+
+        elements = doc.selectXpath("//body");
+        assertEquals(1, elements.size());
+        assertEquals("One", elements.first().id());
+    }
+
+    @ParameterizedTest
+    @MethodSource("provideEvaluators")
+    void cssAndXpathEquivalents(Document doc, String css, String xpath) {
+        Elements fromCss = doc.select(css);
+        Elements fromXpath = doc.selectXpath(xpath);
+
+        assertTrue(fromCss.size() >= 1);
+        assertTrue(fromXpath.size() >= 1);
+        // tests same size, order, and contents
+        assertEquals(fromCss, fromXpath);
+    }
+
+    private static Stream<Arguments> provideEvaluators() {
+        String html = "<div id=1><div id=2><p class=foo>Hello</p></div></div><DIV id=3>";
+        Document doc = Jsoup.parse(html);
+
+        return Stream.of(
+           Arguments.of(doc, "DIV", "//div"),
+           Arguments.of(doc, "div > p.foo", "//div/p[@class]"),
+           Arguments.of(doc, "div + div", "//div/following-sibling::div[1]"),
+           Arguments.of(doc, "p:containsOwn(Hello)", "//p[contains(text(),\"Hello\")]")
+        );
+    }
+
+    @Test
+    public void canSupplyAlternateFactoryImpl() {
+        // previously we had a test to load Saxon and do an XPath 2.0 query. But we know Saxon works and so that's
+        // redundant - really just need to test that an alternate XPath factory can be used
+
+        System.setProperty(XPathFactoryProperty, AlternateXpathFactory.class.getName());
+
+        String xhtml = "<html xmlns='http://www.w3.org/1999/xhtml'><body id='One'><div>hello</div></body></html>";
+        boolean threw = false;
+        try {
+            Document doc = Jsoup.parse(xhtml, Parser.xmlParser());
+            Elements elements = doc.selectXpath("//*:body");
+
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage().contains("Sorry, no can do!"));
+            threw = true;
+        }
+        assertTrue(threw);
+        System.clearProperty(XPathFactoryProperty);
+
+
+    }
+
+    // minimal, no-op implementation class to verify users can load a factory to support XPath 2.0 etc
+    public static class AlternateXpathFactory extends XPathFactory {
+        public AlternateXpathFactory() {
+            super();
+        }
+
+        @Override
+        public boolean isObjectModelSupported(String objectModel) {
+            return true;
+        }
+
+        @Override
+        public void setFeature(String name, boolean value) throws XPathFactoryConfigurationException {
+
+        }
+
+        @Override
+        public boolean getFeature(String name) throws XPathFactoryConfigurationException {
+            return true;
+        }
+
+        @Override
+        public void setXPathVariableResolver(XPathVariableResolver resolver) {
+
+        }
+
+        @Override
+        public void setXPathFunctionResolver(XPathFunctionResolver resolver) {
+
+        }
+
+        @Override
+        public XPath newXPath() {
+            throw new IllegalArgumentException("Sorry, no can do!");
+        }
+    }
+}