Repository: scrapy/scrapy

Issue:
XMLFeedSpider encoding issue
Scrapy version: 1.04

My spider (from the XMLFeedSpider example in the docs) doesn´t seem to read the defined itertag since it contains the iso-8859-1 letter "Þ"
http://www.w3schools.com/charsets/ref_html_8859.asp

I´ve tried my code on a different url with english xml tags and it works fine.

``` python
# -*- coding: utf-8 -*-

import scrapy

from scrapy.spiders import XMLFeedSpider
from althingi_scraper.items import PartyItem

class PartySpider(XMLFeedSpider):
    name = 'party'
    allowed_domains = ['http://www.althingi.is']
    #session = '145'  

    start_urls = [
        #'http://www.althingi.is/altext/xml/thingflokkar/?lthing=%s' % session,        
        'http://www.althingi.is/altext/xml/thingflokkar/'
    ]   

    itertag = 'þingflokkar'

    def parse_node(self, response, node):        
        item = PartyItem()
        item['party_id'] = node.xpath('@id').extract()
        item['name'] = node.xpath('heiti').extract()
        #item['short_abbr'] = node.xpath('stuttskammstöfun').extract()
        #item['long_abbr'] = node.xpath('löngskammstöfun').extract()

        return item
```

Any thoughts?



Discussion:
Using xml as an iterator solved it. I´ll keep the issue open since this might be an issue with iternode.

EDIT:
I was wrong. It halts after the first node.

Here is the xml markup I´m trying to parse, just for reference.

``` xml
<þingflokkar>
  <þingflokkur id="44">
    <heiti>Björt framtíð</heiti>
    <skammstafanir>
      <stuttskammstöfun>Bf</stuttskammstöfun>
      <löngskammstöfun>Björt fr.</löngskammstöfun>
    </skammstafanir>
    <tímabil>
      <fyrstaþing>142</fyrstaþing>
    </tímabil>
  </þingflokkur>
  <þingflokkur id="2">
    <heiti>Framsóknarflokkur</heiti>
    <skammstafanir>
      <stuttskammstöfun>F</stuttskammstöfun>
      <löngskammstöfun>Framsfl.</löngskammstöfun>
    </skammstafanir>
    <tímabil>
      <fyrstaþing>29</fyrstaþing>
    </tímabil>
  </þingflokkur>
  <þingflokkur id="43">
    <heiti>Píratar</heiti>
    <skammstafanir>
      <stuttskammstöfun>P</stuttskammstöfun>
      <löngskammstöfun>Píratar</löngskammstöfun>
    </skammstafanir>
    <tímabil>
      <fyrstaþing>142</fyrstaþing>
    </tímabil>
  </þingflokkur>
</þingflokkar>
```


Resolving patch:
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -17,7 +17,7 @@
 
 def xmliter(obj, nodename):
     """Return a iterator of Selector's over all nodes of a XML document,
-       given tha name of the node to iterate. Useful for parsing XML feeds.
+       given the name of the node to iterate. Useful for parsing XML feeds.
 
     obj can be:
     - a Response object
@@ -35,7 +35,7 @@ def xmliter(obj, nodename):
     header_end = re_rsearch(HEADER_END_RE, text)
     header_end = text[header_end[1]:].strip() if header_end else ''
 
-    r = re.compile(r"<{0}[\s>].*?</{0}>".format(nodename_patt), re.DOTALL)
+    r = re.compile(r'<%(np)s[\s>].*?</%(np)s>' % {'np': nodename_patt}, re.DOTALL)
     for match in r.finditer(text):
         nodetext = header_start + match.group() + header_end
         yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
@@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
     iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
     selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename)
     for _, node in iterable:
-        nodetext = etree.tostring(node)
+        nodetext = etree.tostring(node, encoding='unicode')
         node.clear()
         xs = Selector(text=nodetext, type='xml')
         if namespace:
@@ -128,8 +128,11 @@ def _getrow(csv_r):
 
 
 def _body_or_str(obj, unicode=True):
-    assert isinstance(obj, (Response, six.string_types, bytes)), \
-        "obj must be Response or basestring, not %s" % type(obj).__name__
+    expected_types = (Response, six.text_type, six.binary_type)
+    assert isinstance(obj, expected_types), \
+        "obj must be %s, not %s" % (
+            " or ".join(t.__name__ for t in expected_types),
+            type(obj).__name__)
     if isinstance(obj, Response):
         if not unicode:
             return obj.body


Test patch:
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import os
 import six
 from twisted.trial import unittest
@@ -46,6 +47,60 @@ def test_xmliter_unusual_node(self):
                  for e in self.xmliter(response, 'matchme...')]
         self.assertEqual(nodenames, [['matchme...']])
 
+    def test_xmliter_unicode(self):
+        # example taken from https://github.com/scrapy/scrapy/issues/1665
+        body = u"""<?xml version="1.0" encoding="UTF-8"?>
+            <þingflokkar>
+               <þingflokkur id="26">
+                  <heiti />
+                  <skammstafanir>
+                     <stuttskammstöfun>-</stuttskammstöfun>
+                     <löngskammstöfun />
+                  </skammstafanir>
+                  <tímabil>
+                     <fyrstaþing>80</fyrstaþing>
+                  </tímabil>
+               </þingflokkur>
+               <þingflokkur id="21">
+                  <heiti>Alþýðubandalag</heiti>
+                  <skammstafanir>
+                     <stuttskammstöfun>Ab</stuttskammstöfun>
+                     <löngskammstöfun>Alþb.</löngskammstöfun>
+                  </skammstafanir>
+                  <tímabil>
+                     <fyrstaþing>76</fyrstaþing>
+                     <síðastaþing>123</síðastaþing>
+                  </tímabil>
+               </þingflokkur>
+               <þingflokkur id="27">
+                  <heiti>Alþýðuflokkur</heiti>
+                  <skammstafanir>
+                     <stuttskammstöfun>A</stuttskammstöfun>
+                     <löngskammstöfun>Alþfl.</löngskammstöfun>
+                  </skammstafanir>
+                  <tímabil>
+                     <fyrstaþing>27</fyrstaþing>
+                     <síðastaþing>120</síðastaþing>
+                  </tímabil>
+               </þingflokkur>
+            </þingflokkar>"""
+
+        for r in (
+            # with bytes
+            XmlResponse(url="http://example.com", body=body.encode('utf-8')),
+            # Unicode body needs encoding information
+            XmlResponse(url="http://example.com", body=body, encoding='utf-8')):
+
+            attrs = []
+            for x in self.xmliter(r, u'þingflokkur'):
+                attrs.append((x.xpath('@id').extract(),
+                              x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
+                              x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
+
+            self.assertEqual(attrs,
+                             [([u'26'], [u'-'], [u'80']),
+                              ([u'21'], [u'Ab'], [u'76']),
+                              ([u'27'], [u'A'], [u'27'])])
 
     def test_xmliter_text(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
@@ -96,6 +151,10 @@ def test_xmliter_exception(self):
 
         self.assertRaises(StopIteration, next, iter)
 
+    def test_xmliter_objtype_exception(self):
+        i = self.xmliter(42, 'product')
+        self.assertRaises(AssertionError, next, i)
+
     def test_xmliter_encoding(self):
         body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
         response = XmlResponse('http://www.example.com', body=body)
@@ -169,6 +228,9 @@ def test_xmliter_namespaces_prefix(self):
         node = next(my_iter)
         self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])
 
+    def test_xmliter_objtype_exception(self):
+        i = self.xmliter(42, 'product')
+        self.assertRaises(TypeError, next, i)
 
 class UtilsCsvTestCase(unittest.TestCase):
     sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')