Repository: scrapy/scrapy
Pull request #1661 (2016-01-06 20:36:31)

[MRG+1] PY3: port utils/iterators
I was getting these errors
`_csv.Error: iterator should return strings, not bytes (did you open the file in text mode?)`
and it seems to be about returning strings on `next()` calls to `csv.reader`'s first parameter

https://docs.python.org/3/library/csv.html#csv.reader:

> csvfile can be any object which supports the iterator protocol and returns a string each time its __next__() method is called

so I ended up using `StringIO` instead of `BytesIO`

codecov-io: ## [Current coverage][1] is `82.94%`
> Merging **#1661** into **master** will increase coverage by **+0.01%** as of [`3b11817`][3]



[1]: https://codecov.io/github/scrapy/scrapy?ref=3b11817006badfb85d8c0c5fc62fde348609d479
[2]: https://codecov.io/github/scrapy/scrapy/features/suggestions?ref=3b11817006badfb85d8c0c5fc62fde348609d479
[3]: https://codecov.io/github/scrapy/scrapy/commit/3b11817006badfb85d8c0c5fc62fde348609d479
[4]: https://codecov.io/github/scrapy/scrapy/compare/95e8ff8ba1dff3ec045dce931b6ea4314e887399...3b11817006badfb85d8c0c5fc62fde348609d479

> Powered by [Codecov](https://codecov.io). Updated on successful CI builds.

Patch:
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 3569cbf37..2b118bfbd 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -106,10 +106,10 @@ Example of shell session
 ========================
 
 Here's an example of a typical shell session where we start by scraping the
-http://scrapy.org page, and then proceed to scrape the http://reddit.com
-page. Finally, we modify the (Reddit) request method to POST and re-fetch it
-getting an error. We end the session by typing Ctrl-D (in Unix systems) or
-Ctrl-Z in Windows.
+http://scrapy.org page, and then proceed to scrape the http://slashdot.org
+page. Finally, we modify the (Slashdot) request method to POST and re-fetch it
+getting a HTTP 405 (method not allowed) error. We end the session by typing
+Ctrl-D (in Unix systems) or Ctrl-Z in Windows.
 
 Keep in mind that the data extracted here may not be the same when you try it,
 as those pages are not static and could have changed by the time you test this.
@@ -140,24 +140,24 @@ all start with the ``[s]`` prefix)::
 
 After that, we can start playing with the objects::
 
-    >>> response.xpath('//title/text()').extract_first()
-    u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
+    >>> response.xpath("//h1/text()").extract()[0]
+    u'Meet Scrapy'
 
-    >>> fetch("http://reddit.com")
+    >>> fetch("http://slashdot.org")
     [s] Available Scrapy objects:
-    [s]   crawler    <scrapy.crawler.Crawler object at 0x7fb3ed9c9c90>
+    [s]   crawler    <scrapy.crawler.Crawler object at 0x1a13b50>
     [s]   item       {}
-    [s]   request    <GET http://reddit.com>
-    [s]   response   <200 https://www.reddit.com/>
-    [s]   settings   <scrapy.settings.Settings object at 0x7fb3ed9c9c10>
-    [s]   spider     <DefaultSpider 'default' at 0x7fb3ecdd3390>
+    [s]   request    <GET http://slashdot.org>
+    [s]   response   <200 http://slashdot.org>
+    [s]   settings   <scrapy.settings.Settings object at 0x2bfd650>
+    [s]   spider     <Spider 'default' at 0x20c6f50>
     [s] Useful shortcuts:
     [s]   shelp()           Shell help (print this help)
     [s]   fetch(req_or_url) Fetch request (or URL) and update local objects
     [s]   view(response)    View response in a browser
 
     >>> response.xpath('//title/text()').extract()
-    [u'reddit: the front page of the internet']
+    [u'Slashdot: News for nerds, stuff that matters']
 
     >>> request = request.replace(method="POST")
 
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index c0d93f7a9..ce59c9719 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,12 +1,11 @@
 import re
 import csv
 import logging
-
 try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
     from io import BytesIO
-
+from io import StringIO
 import six
 
 from scrapy.http import TextResponse, Response
@@ -65,7 +64,7 @@ class _StreamReader(object):
             self._text, self.encoding = obj.body, obj.encoding
         else:
             self._text, self.encoding = obj, 'utf-8'
-        self._is_unicode = isinstance(self._text, unicode)
+        self._is_unicode = isinstance(self._text, six.text_type)
 
     def read(self, n=65535):
         self.read = self._read_unicode if self._is_unicode else self._read_string
@@ -94,7 +93,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
     headers is an iterable that when provided offers the keys
     for the returned dictionaries, if not the first row is used.
-    
+
     quotechar is the character used to enclosure fields on the given obj.
     """
 
@@ -102,7 +101,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     def _getrow(csv_r):
         return [to_unicode(field, encoding) for field in next(csv_r)]
 
-    lines = BytesIO(_body_or_str(obj, unicode=False))
+    # Python 3 csv reader input object needs to return strings
+    if six.PY3:
+        lines = StringIO(_body_or_str(obj, unicode=True))
+    else:
+        lines = BytesIO(_body_or_str(obj, unicode=False))
 
     kwargs = {}
     if delimiter: kwargs["delimiter"] = delimiter
@@ -125,7 +128,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
 
 def _body_or_str(obj, unicode=True):
-    assert isinstance(obj, (Response, six.string_types)), \
+    assert isinstance(obj, (Response, six.string_types, bytes)), \
         "obj must be Response or basestring, not %s" % type(obj).__name__
     if isinstance(obj, Response):
         if not unicode:


Test patch:
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 55ed75c92..015578e1e 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -16,7 +16,6 @@ tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_proxy_connect.py
 tests/test_spidermiddleware_httperror.py
-tests/test_utils_iterators.py
 tests/test_utils_template.py
 tests/test_webclient.py
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 590c53302..d42ed2c91 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,4 +1,5 @@
 import os
+import six
 from twisted.trial import unittest
 
 from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
@@ -99,7 +100,7 @@ class XmliterTestCase(unittest.TestCase):
         body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
-            self.xmliter(response, 'item').next().extract(),
+            next(self.xmliter(response, 'item')).extract(),
             u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
         )
 
@@ -189,11 +190,11 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
-            self.assert_(all((isinstance(k, unicode) for k in result_row.keys())))
-            self.assert_(all((isinstance(v, unicode) for v in result_row.values())))
+            self.assert_(all((isinstance(k, six.text_type) for k in result_row.keys())))
+            self.assert_(all((isinstance(v, six.text_type) for v in result_row.values())))
 
     def test_csviter_delimiter(self):
-        body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t')
+        body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
         response = TextResponse(url="http://example.com/", body=body)
         csv = csviter(response, delimiter='\t')
 
@@ -205,8 +206,8 @@ class UtilsCsvTestCase(unittest.TestCase):
 
     def test_csviter_quotechar(self):
         body1 = get_testdata('feeds', 'feed-sample6.csv')
-        body2 = get_testdata('feeds', 'feed-sample6.csv').replace(",", '|')
-        
+        body2 = get_testdata('feeds', 'feed-sample6.csv').replace(b',', b'|')
+
         response1 = TextResponse(url="http://example.com/", body=body1)
         csv1 = csviter(response1, quotechar="'")
 
@@ -237,7 +238,7 @@ class UtilsCsvTestCase(unittest.TestCase):
                           {u"'id'": u"4",   u"'name'": u"'empty'",   u"'value'": u""}])
 
     def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
-        body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t')
+        body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
         response = Response(url="http://example.com/", body=body)
         csv = csviter(response, delimiter='\t')
 
@@ -249,10 +250,10 @@ class UtilsCsvTestCase(unittest.TestCase):
 
     def test_csviter_headers(self):
         sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
-        headers, body = sample[0].split(','), '\n'.join(sample[1:])
+        headers, body = sample[0].split(b','), b'\n'.join(sample[1:])
 
         response = TextResponse(url="http://example.com/", body=body)
-        csv = csviter(response, headers=headers)
+        csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
 
         self.assertEqual([row for row in csv],
                          [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
@@ -262,7 +263,7 @@ class UtilsCsvTestCase(unittest.TestCase):
 
     def test_csviter_falserow(self):
         body = get_testdata('feeds', 'feed-sample3.csv')
-        body = '\n'.join((body, 'a,b', 'a,b,c,d'))
+        body = b'\n'.join((body, b'a,b', b'a,b,c,d'))
 
         response = TextResponse(url="http://example.com/", body=body)
         csv = csviter(response)