Repository: scrapy/scrapy
Pull request #1533 (2015-10-07 12:06:16)

[MRG+1] xmliter nodename with dot
A dot is a valid character for an xml node name.
See http://www.w3.org/TR/2008/REC-xml-20081126/#NT-NameChar
Trying to match it leads to matching any character because it's not escaped.

`nodename` was treated like a regex which is a bug in itself
since it contradicts the function's doctring.

I'd rather discard the additional test.
Should I?


Patch:
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 273cae0f8..688c2b715 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -146,12 +146,12 @@ If you want to extract only first matched element, you can call the selector ``.
 
 It returns ``None`` if no element was found:
 
-    >>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None
+    >>> response.xpath('//div/[id="not-exists"]/text()').extract_first() is None
     True
 
 A default return value can be provided as an argument, to be used instead of ``None``:
 
-    >>> sel.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found')
+    >>> sel.xpath('//div/[id="not-exists"]/text()').extract_first(default='not-found')
     'not-found'
 
 Notice that CSS selectors can select text or attribute nodes using CSS3
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index b9f682431..c0d93f7a9 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -25,8 +25,10 @@ def xmliter(obj, nodename):
     - a unicode string
     - a string encoded as utf-8
     """
-    HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S)
-    HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S)
+    nodename_patt = re.escape(nodename)
+
+    HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S)
+    HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S)
     text = _body_or_str(obj)
 
     header_start = re.search(HEADER_START_RE, text)
@@ -34,7 +36,7 @@ def xmliter(obj, nodename):
     header_end = re_rsearch(HEADER_END_RE, text)
     header_end = text[header_end[1]:].strip() if header_end else ''
 
-    r = re.compile(r"<%s[\s>].*?</%s>" % (nodename, nodename), re.DOTALL)
+    r = re.compile(r"<{0}[\s>].*?</{0}>".format(nodename_patt), re.DOTALL)
     for match in r.finditer(text):
         nodetext = header_start + match.group() + header_end
         yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]


Test patch:
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index f2780dcf1..590c53302 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -33,6 +33,19 @@ class XmliterTestCase(unittest.TestCase):
         self.assertEqual(attrs,
                          [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
 
+    def test_xmliter_unusual_node(self):
+        body = b"""<?xml version="1.0" encoding="UTF-8"?>
+            <root>
+                <matchme...></matchme...>
+                <matchmenot></matchmenot>
+            </root>
+        """
+        response = XmlResponse(url="http://example.com", body=body)
+        nodenames = [e.xpath('name()').extract()
+                 for e in self.xmliter(response, 'matchme...')]
+        self.assertEqual(nodenames, [['matchme...']])
+
+
     def test_xmliter_text(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""