Repository: scrapy/scrapy
Pull request #1386 (2015-07-25 16:48:25)

[MRG+1] [py3] replace rfc822 with email.utils
@kmike, @dangra 
nice sprint :)
i'm probably already too late with this

kmike: Looks good, thanks!
kmike: +1 to merge it once response patching is removed
nyov: Aye, gotcha. `body_as_unicode()` had slipped my memory.
But I split that commit from this tree, I rewrote the tests too much, and put it here #1391 for some feedback.
kmike: LGTM. I'll leave it to @dangra to merge because he was working on porting cookie code to Python 3.
nyov: Just to make sure, but you probably noticed, I put this on top of the other `tmp-py3` branch. It likely fails the py3 tests on master.
If I should remove the cookie commit again, let me know.
kmike: I'm brave enough to merge this PR by myself if there are no cookie changes (I've checked the related modules and tests, and they look fine after your changes). Daniel said he was working on cookies, so it is better to wait for him if the PR contains PY3 cookie changes.

Patch:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 250f46ad8..a449793c9 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -7,9 +7,9 @@ See documentation in topics/media-pipeline.rst
 import hashlib
 import os
 import os.path
-import rfc822
 import time
 import logging
+from email.utils import parsedate_tz, mktime_tz
 from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 import six
@@ -91,8 +91,8 @@ class S3FilesStore(object):
         def _onsuccess(boto_key):
             checksum = boto_key.etag.strip('"')
             last_modified = boto_key.last_modified
-            modified_tuple = rfc822.parsedate_tz(last_modified)
-            modified_stamp = int(rfc822.mktime_tz(modified_tuple))
+            modified_tuple = parsedate_tz(last_modified)
+            modified_stamp = int(mktime_tz(modified_tuple))
             return {'checksum': checksum, 'last_modified': modified_stamp}
 
         return self._get_boto_key(path).addCallback(_onsuccess)
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 7c017feef..c212f5706 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -92,9 +92,9 @@ class ResponseTypes(object):
         chunk = body[:5000]
         if isbinarytext(chunk):
             return self.from_mimetype('application/octet-stream')
-        elif "<html>" in chunk.lower():
+        elif b"<html>" in chunk.lower():
             return self.from_mimetype('text/html')
-        elif "<?xml" in chunk.lower():
+        elif b"<?xml" in chunk.lower():
             return self.from_mimetype('text/xml')
         else:
             return self.from_mimetype('text')
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 7a3f20476..8f3824abf 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -97,7 +97,7 @@ def iter_errback(iterable, errback, *a, **kw):
     iterating it.
     """
     it = iter(iterable)
-    while 1:
+    while True:
         try:
             yield next(it)
         except StopIteration:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index c269b7f74..4215e41d2 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -106,7 +106,7 @@ def md5sum(file):
     '784406af91dd5a54fbb9c84c2236595a'
     """
     m = hashlib.md5()
-    while 1:
+    while True:
         d = file.read(8096)
         if not d:
             break


Test patch:
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 4dfab4a9a..84bd01c0e 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -12,7 +12,6 @@ tests/test_crawler.py
 tests/test_downloader_handlers.py
 tests/test_downloadermiddleware_ajaxcrawlable.py
 tests/test_downloadermiddleware_cookies.py
-tests/test_downloadermiddleware_decompression.py
 tests/test_downloadermiddleware_defaultheaders.py
 tests/test_downloadermiddleware_downloadtimeout.py
 tests/test_downloadermiddleware_httpauth.py
@@ -28,10 +27,8 @@ tests/test_engine.py
 tests/test_http_cookies.py
 tests/test_logformatter.py
 tests/test_mail.py
-tests/test_middleware.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
-tests/test_pipeline_media.py
 tests/test_proxy_connect.py
 tests/test_responsetypes.py
 tests/test_selector_csstranslator.py
diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py
index 9143611fc..77b35a8c3 100644
--- a/tests/test_downloadermiddleware_decompression.py
+++ b/tests/test_downloadermiddleware_decompression.py
@@ -39,7 +39,7 @@ class DecompressionMiddlewareTest(TestCase):
         assert_samelines(self, new.body, rsp.body)
 
     def test_empty_response(self):
-        rsp = Response(url='http://test.com', body='')
+        rsp = Response(url='http://test.com', body=b'')
         new = self.mw.process_response(None, rsp, self.spider)
         assert new is rsp
         assert not rsp.body
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 48131462c..b6d885330 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -3,6 +3,7 @@ from twisted.trial import unittest
 from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 from scrapy.middleware import MiddlewareManager
+import six
 
 class M1(object):
 
@@ -65,12 +66,20 @@ class MiddlewareManagerTest(unittest.TestCase):
 
     def test_methods(self):
         mwman = TestMiddlewareManager(M1(), M2(), M3())
-        self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
-            [M1, M2])
-        self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
-            [M2, M1])
-        self.assertEqual([x.im_class for x in mwman.methods['process']],
-            [M1, M3])
+        if six.PY2:
+            self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
+                [M1, M2])
+            self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
+                [M2, M1])
+            self.assertEqual([x.im_class for x in mwman.methods['process']],
+                [M1, M3])
+        else:
+            self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
+                [M1, M2])
+            self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
+                [M2, M1])
+            self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
+                [M1, M3])
 
     def test_enabled(self):
         m1, m2, m3 = M1(), M2(), M3()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index b12f41174..ac0438eba 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -192,7 +192,7 @@ def _create_item_with_files(*files):
 def _prepare_request_object(item_url):
     return Request(
         item_url,
-        meta={'response': Response(item_url, status=200, body='data')})
+        meta={'response': Response(item_url, status=200, body=b'data')})
 
 
 if __name__ == "__main__":
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index fd8b28ce1..f30b4fea3 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -44,7 +44,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
     def test_default_media_downloaded(self):
         request = Request('http://url')
-        response = Response('http://url', body='')
+        response = Response('http://url', body=b'')
         assert self.pipe.media_downloaded(response, request, self.info) is response
 
     def test_default_media_failed(self):
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index b4083403a..1d78d0976 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -51,7 +51,7 @@ class ResponseTypesTest(unittest.TestCase):
         for source, cls in mappings:
             retcls = responsetypes.from_body(source)
             assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls)
-        
+
     def test_from_headers(self):
         mappings = [
             ({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse),