Repository: scrapy/parsel

Issue:
parsel-cli - parsel as a cli application
I've been working small cli parsel wrapper for iterpreting css and xpath selectiors (inspired by `scrapy shell`).

https://github.com/granitosaurus/parsel-cli

It puts you straight into css or xpath interpreter mode (or embed python shell) and evaluates input css/xpath selectors using parsel.

I think it's better off as standalone tool but maybe it's worth mentioning somewhere in readme :)

Discussion:

Patch:
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 2b95f23..7052527 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.5.2
+current_version = 1.5.1
 commit = True
 tag = True
 tag_name = v{new_version}
diff --git a/NEWS b/NEWS
index 0b65faa..848542a 100644
--- a/NEWS
+++ b/NEWS
@@ -3,16 +3,6 @@
 History
 -------
 
-1.5.2 (2019-08-09)
-~~~~~~~~~~~~~~~~~~
-
-* ``Selector.remove_namespaces`` received a significant performance improvement
-* The value of ``data`` within the printable representation of a selector
-  (``repr(selector)``) now ends in ``...`` when truncated, to make the
-  truncation obvious.
-* Minor documentation improvements.
-
-
 1.5.1 (2018-10-25)
 ~~~~~~~~~~~~~~~~~~
 
@@ -22,7 +12,6 @@ History
 * documentation improvements;
 * Python 3.7 tests are run on CI; other test improvements.
 
-
 1.5.0 (2018-07-04)
 ~~~~~~~~~~~~~~~~~~
 
diff --git a/README.rst b/README.rst
index 1ce4f3a..af5bccb 100644
--- a/README.rst
+++ b/README.rst
@@ -24,7 +24,6 @@ Features
 --------
 
 * Extract text using CSS or XPath selectors
-* Remove elements using CSS or XPath selectors
 * Regular expression helper methods
 
 Example::
@@ -36,7 +35,7 @@ Example::
                 <ul>
                     <li><a href="http://example.com">Link 1</a></li>
                     <li><a href="http://scrapy.org">Link 2</a></li>
-                </ul>
+                </ul
             </body>
             </html>""")
     >>>
diff --git a/docs/usage.rst b/docs/usage.rst
index c837fbd..5663033 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -385,41 +385,6 @@ XPath specification.
 .. _Location Paths: https://www.w3.org/TR/xpath#location-paths
 
 
-Removing elements
------------------
-
-If for any reason you need to remove elements based on a Selector or
-a SelectorList, you can do it with the ``remove()`` method, available for both
-classes.
-
-.. warning:: this is a destructive action and cannot be undone. The original
-    content of the selector is removed from the elements tree. This could be useful
-    when trying to reduce the memory footprint of Responses.
-
-Example removing an ad from a blog post:
-
-    >>> from parsel import Selector
-    >>> doc = u"""
-    ... <article>
-    ...     <div class="row">Content paragraph...</div>
-    ...     <div class="row">
-    ...         <div class="ad">
-    ...             Ad content...
-    ...             <a href="http://...">Link</a>
-    ...         </div>
-    ...     </div>
-    ...     <div class="row">More content...</div>
-    ... </article>
-    ... """
-    >>> sel = Selector(text=doc)
-    >>> sel.xpath('//div/text()').getall()
-    ['Content paragraph...', 'Ad content...', 'Link', 'More content...']
-    >>> sel.xpath('//div[@class="ad"]').remove()
-    >>> sel.xpath('//div//text()').getall()
-    ['Content paragraph...', 'More content...']
-    >>>
-
-
 Using EXSLT extensions
 ----------------------
 
@@ -565,8 +530,6 @@ inside another ``itemscope``.
 .. _regular expressions: http://exslt.org/regexp/index.html
 .. _set manipulation: http://exslt.org/set/index.html
 
-.. _topics-xpath-other-extensions:
-
 Other XPath extensions
 ----------------------
 
@@ -780,6 +743,23 @@ are more predictable: ``.get()`` always returns a single result,
 ``.getall()`` always returns a list of all extracted results.
 
 
+Command-Line Interface Tools
+============================
+
+There are third-party tools that allow using Parsel from the command line:
+
+-   `Parsel CLI <https://github.com/rmax/parsel-cli>`_ allows applying
+    Parsel selectors to the standard input. For example, you can apply a Parsel
+    selector to the output of cURL_.
+
+-   `parselcli
+    <https://github.com/Granitosaurus/parsel-cli>`_ provides an interactive
+    shell that allows applying Parsel selectors to a remote URL or a local
+    file.
+
+.. _cURL: https://curl.haxx.se/
+
+
 .. _topics-selectors-ref:
 
 API reference
@@ -949,7 +929,6 @@ You can pass several namespaces (here we're using shorter 1-letter prefixes)::
      '//lh3.googleusercontent.com/-7xisiK0EArc/AAAAAAAAAAI/AAAAAAAAAuM/-r6o6A8RKCM/s512-c/photo.jpg',
      ...
 
-.. _topics-xpath-variables:
 
 Variables in XPath expressions
 ------------------------------
diff --git a/parsel/__init__.py b/parsel/__init__.py
index b975367..e9eb5a5 100644
--- a/parsel/__init__.py
+++ b/parsel/__init__.py
@@ -5,7 +5,7 @@ or CSS selectors
 
 __author__ = 'Scrapy project'
 __email__ = 'info@scrapy.org'
-__version__ = '1.5.2'
+__version__ = '1.5.1'
 
 from parsel.selector import Selector, SelectorList  # NOQA
 from parsel.csstranslator import css2xpath  # NOQA
diff --git a/parsel/selector.py b/parsel/selector.py
index ac52853..f9292a4 100644
--- a/parsel/selector.py
+++ b/parsel/selector.py
@@ -7,18 +7,10 @@ import sys
 import six
 from lxml import etree, html
 
-from .utils import flatten, iflatten, extract_regex, shorten
+from .utils import flatten, iflatten, extract_regex
 from .csstranslator import HTMLTranslator, GenericTranslator
 
 
-class CannotRemoveElementWithoutRoot(Exception):
-    pass
-
-
-class CannotRemoveElementWithoutParent(Exception):
-    pass
-
-
 class SafeXMLParser(etree.XMLParser):
     def __init__(self, *args, **kwargs):
         kwargs.setdefault('resolve_entities', False)
@@ -158,13 +150,6 @@ class SelectorList(list):
         else:
             return {}
 
-    def remove(self):
-        """
-        Remove matched nodes from the parent for each element in this list.
-        """
-        for x in self:
-            x.remove()
-
 
 class Selector(object):
     """
@@ -273,8 +258,6 @@ class Selector(object):
 
         In the background, CSS queries are translated into XPath queries using
         `cssselect`_ library and run ``.xpath()`` method.
-        
-        .. _cssselect: https://pypi.python.org/pypi/cssselect/
         """
         return self.xpath(self._css2xpath(query))
 
@@ -354,32 +337,8 @@ class Selector(object):
             for an in el.attrib.keys():
                 if an.startswith('{'):
                     el.attrib[an.split('}', 1)[1]] = el.attrib.pop(an)
-        # remove namespace declarations
-        etree.cleanup_namespaces(self.root)
-
-    def remove(self):
-        """
-        Remove matched nodes from the parent element.
-        """
-        try:
-            parent = self.root.getparent()
-        except AttributeError:
-            # 'str' object has no attribute 'getparent'
-            raise CannotRemoveElementWithoutRoot(
-                "The node you're trying to remove has no root, "
-                "are you trying to remove a pseudo-element? "
-                "Try to use 'li' as a selector instead of 'li::text' or "
-                "'//li' instead of '//li/text()', for example."
-            )
-
-        try:
-            parent.remove(self.root)
-        except AttributeError:
-            # 'NoneType' object has no attribute 'remove'
-            raise CannotRemoveElementWithoutParent(
-                "The node you're trying to remove has no parent, "
-                "are you trying to remove a root element?"
-            )
+            # remove namespace declarations
+            etree.cleanup_namespaces(self.root)
 
     @property
     def attrib(self):
@@ -397,6 +356,6 @@ class Selector(object):
     __nonzero__ = __bool__
 
     def __str__(self):
-        data = repr(shorten(self.get(), width=40))
+        data = repr(self.get()[:40])
         return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data)
     __repr__ = __str__
diff --git a/parsel/utils.py b/parsel/utils.py
index 458bc6c..56bb105 100644
--- a/parsel/utils.py
+++ b/parsel/utils.py
@@ -80,15 +80,4 @@ def extract_regex(regex, text, replace_entities=True):
     strings = flatten(strings)
     if not replace_entities:
         return strings
-    return [w3lib_replace_entities(s, keep=['lt', 'amp']) for s in strings]
-
-
-def shorten(text, width, suffix='...'):
-    """Truncate the given text to fit in the given width."""
-    if len(text) <= width:
-        return text
-    if width > len(suffix):
-        return text[:width-len(suffix)] + suffix
-    if width >= 0:
-        return suffix[len(suffix)-width:]
-    raise ValueError('width must be equal or greater than 0')
+    return [w3lib_replace_entities(s, keep=['lt', 'amp']) for s in strings]
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 73aac2a..53f6a1c 100644
--- a/setup.py
+++ b/setup.py
@@ -27,8 +27,7 @@ def has_environment_marker_platform_impl_support():
 
 install_requires = [
     'w3lib>=1.19.0',
-    'lxml;python_version!="3.4"',
-    'lxml<=4.3.5;python_version=="3.4"',
+    'lxml>=2.3',
     'six>=1.5.2',
     'cssselect>=0.9'
 ]
@@ -42,7 +41,7 @@ else:
 
 setup(
     name='parsel',
-    version='1.5.2',
+    version='1.5.1',
     description="Parsel is a library to extract data from HTML and XML using XPath and CSS selectors",
     long_description=readme + '\n\n' + history,
     author="Scrapy project",


Test patch:
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 376b0f7..e504166 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -6,10 +6,6 @@ import unittest
 import pickle
 
 from parsel import Selector
-from parsel.selector import (
-    CannotRemoveElementWithoutRoot,
-    CannotRemoveElementWithoutParent,
-)
 
 
 class SelectorTestCase(unittest.TestCase):
@@ -137,9 +133,9 @@ class SelectorTestCase(unittest.TestCase):
         body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')
         sel = self.sscls(text=body)
 
-        representation = "<Selector xpath='//input/@name' data='{}...'>".format(37 * 'b')
+        representation = "<Selector xpath='//input/@name' data='{}'>".format(40 * 'b')
         if six.PY2:
-            representation = "<Selector xpath='//input/@name' data=u'{}...'>".format(37 * 'b')
+            representation = "<Selector xpath='//input/@name' data=u'{}'>".format(40 * 'b')
 
         self.assertEqual(
             [repr(it) for it in sel.xpath('//input/@name')],
@@ -629,61 +625,28 @@ class SelectorTestCase(unittest.TestCase):
     def test_remove_namespaces(self):
         xml = u"""<?xml version="1.0" encoding="UTF-8"?>
 <feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en-US" xmlns:media="http://search.yahoo.com/mrss/">
-  <link type="text/html"/>
-  <entry>
-    <link type="text/html"/>
-  </entry>
-  <link type="application/atom+xml"/>
+  <link type="text/html">
+  <link type="application/atom+xml">
 </feed>
 """
         sel = self.sscls(text=xml, type='xml')
         self.assertEqual(len(sel.xpath("//link")), 0)
         self.assertEqual(len(sel.xpath("./namespace::*")), 3)
         sel.remove_namespaces()
-        self.assertEqual(len(sel.xpath("//link")), 3)
-        self.assertEqual(len(sel.xpath("./namespace::*")), 1)
-
-    def test_remove_namespaces_embedded(self):
-        xml = u"""
-        <feed xmlns="http://www.w3.org/2005/Atom">
-          <link type="text/html"/>
-          <entry>
-            <link type="text/html"/>
-          </entry>
-          <svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 100 100">
-            <linearGradient id="gradient">
-              <stop class="begin" offset="0%" style="stop-color:yellow;"/>
-              <stop class="end" offset="80%" style="stop-color:green;"/>
-            </linearGradient>
-            <circle cx="50" cy="50" r="30" style="fill:url(#gradient)" />
-          </svg>
-        </feed>
-        """
-        sel = self.sscls(text=xml, type='xml')
-        self.assertEqual(len(sel.xpath("//link")), 0)
-        self.assertEqual(len(sel.xpath("//stop")), 0)
-        self.assertEqual(len(sel.xpath("./namespace::*")), 2)
-        self.assertEqual(len(sel.xpath("//f:link", namespaces={'f': 'http://www.w3.org/2005/Atom'})), 2)
-        self.assertEqual(len(sel.xpath("//s:stop", namespaces={'s': 'http://www.w3.org/2000/svg'})), 2)
-        sel.remove_namespaces()
         self.assertEqual(len(sel.xpath("//link")), 2)
-        self.assertEqual(len(sel.xpath("//stop")), 2)
         self.assertEqual(len(sel.xpath("./namespace::*")), 1)
 
     def test_remove_attributes_namespaces(self):
         xml = u"""<?xml version="1.0" encoding="UTF-8"?>
 <feed xmlns:atom="http://www.w3.org/2005/Atom" xml:lang="en-US" xmlns:media="http://search.yahoo.com/mrss/">
-  <link atom:type="text/html"/>
-  <entry>
-    <link atom:type="text/html"/>
-  </entry>
-  <link atom:type="application/atom+xml"/>
+  <link atom:type="text/html">
+  <link atom:type="application/atom+xml">
 </feed>
 """
         sel = self.sscls(text=xml, type='xml')
         self.assertEqual(len(sel.xpath("//link/@type")), 0)
         sel.remove_namespaces()
-        self.assertEqual(len(sel.xpath("//link/@type")), 3)
+        self.assertEqual(len(sel.xpath("//link/@type")), 2)
 
     def test_smart_strings(self):
         """Lxml smart strings return values"""
@@ -749,57 +712,6 @@ class SelectorTestCase(unittest.TestCase):
         self.assertEqual(u'<html><body><p>Grainy</p></body></html>',
                           self.sscls(text).extract())
 
-    def test_remove_selector_list(self):
-        sel = self.sscls(text=u'<html><body><ul><li>1</li><li>2</li><li>3</li></ul></body></html>')
-        sel_list = sel.css('li')
-        sel_list.remove()
-        self.assertIsInstance(sel.css('li'), self.sscls.selectorlist_cls)
-        self.assertEqual(sel.css('li'), [])
-
-    def test_remove_selector(self):
-        sel = self.sscls(text=u'<html><body><ul><li>1</li><li>2</li><li>3</li></ul></body></html>')
-        sel_list = sel.css('li')
-        sel_list[0].remove()
-        self.assertIsInstance(sel.css('li'), self.sscls.selectorlist_cls)
-        self.assertEqual(sel.css('li::text').getall(), ['2', '3'])
-
-    def test_remove_pseudo_element_selector_list(self):
-        sel = self.sscls(text=u'<html><body><ul><li>1</li><li>2</li><li>3</li></ul></body></html>')
-        sel_list = sel.css('li::text')
-        self.assertEqual(sel_list.getall(), ['1', '2', '3'])
-        with self.assertRaises(CannotRemoveElementWithoutRoot):
-            sel_list.remove()
-
-        self.assertIsInstance(sel.css('li'), self.sscls.selectorlist_cls)
-        self.assertEqual(sel.css('li::text').getall(), ['1', '2', '3'])
-
-    def test_remove_pseudo_element_selector(self):
-        sel = self.sscls(text=u'<html><body><ul><li>1</li><li>2</li><li>3</li></ul></body></html>')
-        sel_list = sel.css('li::text')
-        self.assertEqual(sel_list.getall(), ['1', '2', '3'])
-        with self.assertRaises(CannotRemoveElementWithoutRoot):
-            sel_list[0].remove()
-
-        self.assertIsInstance(sel.css('li'), self.sscls.selectorlist_cls)
-        self.assertEqual(sel.css('li::text').getall(), ['1', '2', '3'])
-
-    def test_remove_root_element_selector(self):
-        sel = self.sscls(text=u'<html><body><ul><li>1</li><li>2</li><li>3</li></ul></body></html>')
-        sel_list = sel.css('li::text')
-        self.assertEqual(sel_list.getall(), ['1', '2', '3'])
-        with self.assertRaises(CannotRemoveElementWithoutParent):
-            sel.remove()
-
-        with self.assertRaises(CannotRemoveElementWithoutParent):
-            sel.css('html').remove()
-
-        self.assertIsInstance(sel.css('li'), self.sscls.selectorlist_cls)
-        self.assertEqual(sel.css('li::text').getall(), ['1', '2', '3'])
-
-        sel.css('body').remove()
-        self.assertEqual(sel.get(), '<html></html>')
-
-
 class ExsltTestCase(unittest.TestCase):
 
     sscls = Selector
diff --git a/tests/test_utils.py b/tests/test_utils.py
deleted file mode 100644
index da20ec2..0000000
--- a/tests/test_utils.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from parsel.utils import shorten
-
-from pytest import mark, raises
-import six
-
-
-@mark.parametrize(
-    'width,expected',
-    (
-        (-1, ValueError),
-        (0, u''),
-        (1, u'.'),
-        (2, u'..'),
-        (3, u'...'),
-        (4, u'f...'),
-        (5, u'fo...'),
-        (6, u'foobar'),
-        (7, u'foobar'),
-    )
-)
-def test_shorten(width, expected):
-    if isinstance(expected, six.string_types):
-        assert shorten(u'foobar', width) == expected
-    else:
-        with raises(expected):
-            shorten(u'foobar', width)