Repository: gawel/pyquery

Issue:
Constructor is potentially dangerous
We use PyQuery internally in our system to parse rendered HTML strings that we store in the database to be displayed in a Django web response. Essentially the code looks a bit like this:

```python
def render_document(document):
    parsed = PyQuery(document.rendered_html_string)
    ...
```

But if untrusted users can cause that `document.rendered_html_string` to become a string that looks something like this:

```
http://internaldomain/api/get_users/dangerous
```

Then, it becomes...

```python
def render_document(document):
    parsed = PyQuery('http://internaldomain/api/get_users/dangerous')
   ...
```
which will cause PyQuery to `requests.get('http://internaldomain/api/get_users/dangerous')` which could be a big security risk. 

It happens because of [the constructor](https://github.com/gawel/pyquery/blob/2351446c56d5c3df93e3fd81a2b828cf00e3b648/pyquery/pyquery.py#L190-L196) being too "naive". 

On our app, we solved that by making the PyQuery constructor wrapped in a piece of code that does something like this:

```python
def safer_pyquery(*args, **kwargs):
    # SIMPLIFIED FOR EXAMPLE
    args[0] = ' ' + args[0]
    return PyQuery(*args, **kwargs)
```

Ideally, we'd love to be able to always invoke `PyQuery` like this:

```python
def render_document(document):
    parsed = PyQuery(html=document.rendered_html_string)
    ...
```
In similar style to how you can do `pq(url='http://google.com/')` as mentioned in the Quickstart documentation. 


Comments:

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 971d377..a0a7d59 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,8 +1,19 @@
+2.0 (unreleased)
+----------------
+
+- Breaking change: inputs starting with ``"http://"`` or ``"https://"`` like
+  ``PyQuery("http://example.com")`` will no longer fetch the contents of the URL.
+  Users desiring the old behavior should switch to ``PyQuery(url="http://example.com")``.
+
 1.4.4 (unreleased)
 ------------------
 
 - Add nextUntil method
 
+- ``.remove()`` no longer inserts a space in place of the removed element
+
+- Fix escaping of top-level element text in ``.html()`` output
+
 
 1.4.3 (2020-11-21)
 ------------------
diff --git a/docs/scrap.rst b/docs/scrap.rst
index 86b0b6d..6a469d6 100644
--- a/docs/scrap.rst
+++ b/docs/scrap.rst
@@ -7,17 +7,17 @@ Scraping
 
 PyQuery is able to load an html document from a url::
 
-  >>> pq(your_url)
+  >>> pq(url=your_url)
   [<html>]
 
 By default it uses python's urllib.
 
 If `requests`_ is installed then it will use it. This allow you to use most of `requests`_ parameters::
 
-  >>> pq(your_url, headers={'user-agent': 'pyquery'})
+  >>> pq(url=your_url, headers={'user-agent': 'pyquery'})
   [<html>]
 
-  >>> pq(your_url, {'q': 'foo'}, method='post', verify=True)
+  >>> pq(url=your_url, data={'q': 'foo'}, method='post', verify=True)
   [<html>]
 
 
diff --git a/pyquery/pyquery.py b/pyquery/pyquery.py
index e549f0e..13afbff 100644
--- a/pyquery/pyquery.py
+++ b/pyquery/pyquery.py
@@ -8,6 +8,7 @@ from urllib.parse import urljoin
 from .openers import url_opener
 from .text import extract_text
 from copy import deepcopy
+from html import escape
 from lxml import etree
 import lxml.html
 import inspect
@@ -149,14 +150,6 @@ class PyQuery(list):
         self._base_url = None
         self.parser = kwargs.pop('parser', None)
 
-        if (len(args) >= 1 and
-                isinstance(args[0], str) and
-                args[0].split('://', 1)[0] in ('http', 'https')):
-            kwargs['url'] = args[0]
-            if len(args) >= 2:
-                kwargs['data'] = args[1]
-            args = []
-
         if 'parent' in kwargs:
             self._parent = kwargs.pop('parent')
         else:
@@ -1085,9 +1078,9 @@ class PyQuery(list):
                 return None
             tag = self[0]
             children = tag.getchildren()
+            html = escape(tag.text or '', quote=False)
             if not children:
-                return tag.text or ''
-            html = tag.text or ''
+                return html
             if 'encoding' not in kwargs:
                 kwargs['encoding'] = str
             html += u''.join([etree.tostring(e, **kwargs)
@@ -1456,7 +1449,7 @@ class PyQuery(list):
              >>> d('strong').remove()
              [<strong>]
              >>> print(d)
-             <div>Maybe <em>she</em> does   know</div>
+             <div>Maybe <em>she</em> does  know</div>
         """
         if expr is no_default:
             for tag in self:
@@ -1467,11 +1460,11 @@ class PyQuery(list):
                         if prev is None:
                             if not parent.text:
                                 parent.text = ''
-                            parent.text += ' ' + tag.tail
+                            parent.text += tag.tail
                         else:
                             if not prev.tail:
                                 prev.tail = ''
-                            prev.tail += ' ' + tag.tail
+                            prev.tail += tag.tail
                     parent.remove(tag)
         else:
             results = self._copy(expr, self)


Test patch:
diff --git a/tests/test_pyquery.py b/tests/test_pyquery.py
index d82a556..1ab40e1 100644
--- a/tests/test_pyquery.py
+++ b/tests/test_pyquery.py
@@ -491,9 +491,9 @@ Bacon</textarea>
         d = pq(self.html)
         d('img').remove()
         val = d('a:first').html()
-        assert val == 'Test My link text', repr(val)
+        assert val == 'TestMy link text', repr(val)
         val = d('a:last').html()
-        assert val == ' My link text 2', repr(val)
+        assert val == 'My link text 2', repr(val)
 
     def test_class(self):
         d = pq('<div></div>')
@@ -534,9 +534,10 @@ Bacon</textarea>
         self.assertEqual(d('#textarea-multi').val(), multi_expected)
         self.assertEqual(d('#textarea-multi').text(), multi_expected)
         multi_new = '''Bacon\n<b>Eggs</b>\nSpam'''
+        multi_new_expected = '''Bacon\n&lt;b&gt;Eggs&lt;/b&gt;\nSpam'''
         d('#textarea-multi').val(multi_new)
-        self.assertEqual(d('#textarea-multi').val(), multi_new)
-        self.assertEqual(d('#textarea-multi').text(), multi_new)
+        self.assertEqual(d('#textarea-multi').val(), multi_new_expected)
+        self.assertEqual(d('#textarea-multi').text(), multi_new_expected)
 
     def test_val_for_select(self):
         d = pq(self.html4)
@@ -622,6 +623,13 @@ Bacon</textarea>
         self.assertEqual(new_html, expected)
         self.assertIn(replacement, new_html)
 
+    def test_html_escape(self):
+        inner_html = 'encoded &lt;script&gt; tag with "quotes".' \
+                     '<span>nested &lt;tag&gt;</span>'
+        html = '<div>' + inner_html + '</div>'
+        d = pq(html)
+        self.assertEqual(d.html(), inner_html)
+
 
 class TestAjax(TestCase):
 
@@ -894,14 +902,14 @@ class TestWebScrapping(TestCase):
         self.application_url = self.s.application_url.rstrip('/')
 
     def test_get(self):
-        d = pq(self.application_url, {'q': 'foo'},
+        d = pq(url=self.application_url, data={'q': 'foo'},
                method='get')
         print(d)
         self.assertIn('REQUEST_METHOD: GET', d('p').text())
         self.assertIn('q=foo', d('p').text())
 
     def test_post(self):
-        d = pq(self.application_url, {'q': 'foo'},
+        d = pq(url=self.application_url, data={'q': 'foo'},
                method='post')
         self.assertIn('REQUEST_METHOD: POST', d('p').text())
         self.assertIn('q=foo', d('p').text())
@@ -911,7 +919,7 @@ class TestWebScrapping(TestCase):
             import requests
             session = requests.Session()
             session.headers.update({'X-FOO': 'bar'})
-            d = pq(self.application_url, {'q': 'foo'},
+            d = pq(url=self.application_url, data={'q': 'foo'},
                    method='get', session=session)
             self.assertIn('HTTP_X_FOO: bar', d('p').text())
         else:
@@ -924,7 +932,7 @@ class TestWebScrapping(TestCase):
 class TestWebScrappingEncoding(TestCase):
 
     def test_get(self):
-        d = pq(u'http://ru.wikipedia.org/wiki/Заглавная_страница',
+        d = pq(url=u'http://ru.wikipedia.org/wiki/Заглавная_страница',
                method='get')
         print(d)
         self.assertEqual(d('#pt-login').text(), u'Войти')
@@ -942,9 +950,9 @@ class TestWebScrappingTimeouts(TestCase):
         self.application_url = self.s.application_url.rstrip('/')
 
     def test_get(self):
-        pq(self.application_url)
+        pq(url=self.application_url)
         with self.assertRaises(Exception):
-            pq(self.application_url, timeout=1)
+            pq(url=self.application_url, timeout=1)
 
     def tearDown(self):
         self.s.shutdown()