Repository: scrapy/scrapy
Pull request #1379 (2015-07-23 16:34:03)

PY3 port scrapy.utils.python


Patch:
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 5d5d5b004..57016811f 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -23,8 +23,12 @@ def flatten(x):
     >>> [1, 2, [3,4], (5,6)]
     [1, 2, [3, 4], (5, 6)]
     >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])
-    [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]"""
-
+    [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]
+    >>> flatten(["foo", "bar"])
+    ['foo', 'bar']
+    >>> flatten(["foo", ["baz", 42], "bar"])
+    ['foo', 'baz', 42, 'bar']
+    """
     return list(iflatten(x))
 
 
@@ -32,15 +36,38 @@ def iflatten(x):
     """iflatten(sequence) -> iterator
 
     Similar to ``.flatten()``, but returns iterator instead"""
-
     for el in x:
-        if hasattr(el, "__iter__"):
+        if is_listlike(el):
             for el_ in flatten(el):
                 yield el_
         else:
             yield el
 
 
+def is_listlike(x):
+    """
+    >>> is_listlike("foo")
+    False
+    >>> is_listlike(5)
+    False
+    >>> is_listlike(b"foo")
+    False
+    >>> is_listlike([b"foo"])
+    True
+    >>> is_listlike((b"foo",))
+    True
+    >>> is_listlike({})
+    True
+    >>> is_listlike(set())
+    True
+    >>> is_listlike((x for x in range(3)))
+    True
+    >>> is_listlike(six.moves.xrange(5))
+    True
+    """
+    return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
+
+
 def unique(list_, key=lambda x: x):
     """efficient function to uniquify a list preserving item order"""
     seen = set()
@@ -115,11 +142,14 @@ def re_rsearch(pattern, text, chunk_size=1024):
             yield (text[offset:], offset)
         yield (text, 0)
 
-    pattern = re.compile(pattern) if isinstance(pattern, basestring) else pattern
+    if isinstance(pattern, six.string_types):
+        pattern = re.compile(pattern)
+
     for chunk, offset in _chunk_iter():
         matches = [match for match in pattern.finditer(chunk)]
         if matches:
-            return (offset + matches[-1].span()[0], offset + matches[-1].span()[1])
+            start, end = matches[-1].span()
+            return offset + start, offset + end
     return None
 
 
@@ -135,14 +165,16 @@ def memoizemethod_noargs(method):
         return cache[self]
     return new_method
 
-_BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"])
+_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
+_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
 
 
 def isbinarytext(text):
-    """Return True if the given text is considered binary, or false
+    """Return True if the given text is considered binary, or False
     otherwise, by looking for binary bytes at their chars
     """
-    assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__
+    if not isinstance(text, bytes):
+        raise TypeError("text must be bytes, got '%s'" % type(text).__name__)
     return any(c in _BINARYCHARS for c in text)
 
 
@@ -246,20 +278,22 @@ class WeakKeyCache(object):
         return self._weakdict[key]
 
 
+@deprecated
 def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
-    """Return a (new) dict with the unicode keys (and values if, keys_only is
+    """Return a (new) dict with unicode keys (and values when "keys_only" is
     False) of the given dict converted to strings. `dct_or_tuples` can be a
     dict or a list of tuples, like any dict constructor supports.
     """
     d = {}
     for k, v in six.iteritems(dict(dct_or_tuples)):
-        k = k.encode(encoding) if isinstance(k, unicode) else k
+        k = k.encode(encoding) if isinstance(k, six.text_type) else k
         if not keys_only:
-            v = v.encode(encoding) if isinstance(v, unicode) else v
+            v = v.encode(encoding) if isinstance(v, six.text_type) else v
         d[k] = v
     return d
 
 
+@deprecated
 def is_writable(path):
     """Return True if the given path can be written (if it exists) or created
     (if it doesn't exist)
@@ -270,6 +304,7 @@ def is_writable(path):
         return os.access(os.path.dirname(path), os.W_OK)
 
 
+@deprecated
 def setattr_default(obj, name, value):
     """Set attribute value, but only if it's not already set. Similar to
     setdefault() for dicts.


Test patch:
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 842217dd0..aa091835c 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -59,7 +59,6 @@ tests/test_utils_defer.py
 tests/test_utils_iterators.py
 tests/test_utils_jsonrpc.py
 tests/test_utils_log.py
-tests/test_utils_python.py
 tests/test_utils_reqser.py
 tests/test_utils_request.py
 tests/test_utils_response.py
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index fa77356de..ca394ebf5 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -2,6 +2,7 @@ import functools
 import operator
 import unittest
 from itertools import count
+import six
 
 from scrapy.utils.python import (
     memoizemethod_noargs, isbinarytext, equal_attributes,
@@ -12,10 +13,10 @@ __doctests__ = ['scrapy.utils.python']
 
 class ToUnicodeTest(unittest.TestCase):
     def test_converting_an_utf8_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode('lel\xc3\xb1e'), u'lel\xf1e')
+        self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')
 
     def test_converting_a_latin_1_encoded_string_to_unicode(self):
-        self.assertEqual(to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e')
+        self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e')
 
     def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
         self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
@@ -23,28 +24,34 @@ class ToUnicodeTest(unittest.TestCase):
     def test_converting_a_strange_object_should_raise_TypeError(self):
         self.assertRaises(TypeError, to_unicode, 423)
 
-    def test_check_errors_argument_works(self):
-        self.assertIn(u'\ufffd', to_unicode('a\xedb', 'utf-8', errors='replace'))
+    def test_errors_argument(self):
+        self.assertEqual(
+            to_unicode(b'a\xedb', 'utf-8', errors='replace'),
+            u'a\ufffdb'
+        )
 
 
 class ToBytesTest(unittest.TestCase):
     def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
-        self.assertEqual(to_bytes(u'\xa3 49'), '\xc2\xa3 49')
+        self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49')
 
     def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
-        self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), '\xa3 49')
+        self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')
 
-    def test_converting_a_regular_string_to_string_should_return_the_same_object(self):
-        self.assertEqual(to_bytes('lel\xf1e'), 'lel\xf1e')
+    def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
+        self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
 
     def test_converting_a_strange_object_should_raise_TypeError(self):
         self.assertRaises(TypeError, to_bytes, unittest)
 
-    def test_check_errors_argument_works(self):
-        self.assertIn('?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace'))
+    def test_errors_argument(self):
+        self.assertEqual(
+            to_bytes(u'a\ufffdb', 'latin-1', errors='replace'),
+            b'a?b'
+        )
 
 
-class UtilsPythonTestCase(unittest.TestCase):
+class MemoizedMethodTest(unittest.TestCase):
     def test_memoizemethod_noargs(self):
         class A(object):
 
@@ -62,19 +69,23 @@ class UtilsPythonTestCase(unittest.TestCase):
         assert one is two
         assert one is not three
 
-    def test_isbinarytext(self):
 
-        # basic tests
-        assert not isbinarytext("hello")
+class IsBinaryTextTest(unittest.TestCase):
+    def test_isbinarytext(self):
+        assert not isbinarytext(b"hello")
 
-        # utf-16 strings contain null bytes
+    def test_utf_16_strings_contain_null_bytes(self):
         assert not isbinarytext(u"hello".encode('utf-16'))
 
-        # one with encoding
-        assert not isbinarytext("<div>Price \xa3</div>")
+    def test_one_with_encoding(self):
+        assert not isbinarytext(b"<div>Price \xa3</div>")
+
+    def test_real_binary_bytes(self):
+        assert isbinarytext(b"\x02\xa3")
 
-        # finally some real binary bytes
-        assert isbinarytext("\x02\xa3")
+
+
+class UtilsPythonTestCase(unittest.TestCase):
 
     def test_equal_attributes(self):
         class Obj:
@@ -134,29 +145,32 @@ class UtilsPythonTestCase(unittest.TestCase):
         del k
         self.assertFalse(len(wk._weakdict))
 
+    @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict(self):
-        d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
+        d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
         d2 = stringify_dict(d, keys_only=False)
         self.assertEqual(d, d2)
-        self.failIf(d is d2) # shouldn't modify in place
-        self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
-        self.failIf(any(isinstance(x, unicode) for x in d2.values()))
+        self.failIf(d is d2)  # shouldn't modify in place
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
 
+    @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_tuples(self):
         tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
         d = dict(tuples)
         d2 = stringify_dict(tuples, keys_only=False)
         self.assertEqual(d, d2)
-        self.failIf(d is d2) # shouldn't modify in place
-        self.failIf(any(isinstance(x, unicode) for x in d2.keys()), d2.keys())
-        self.failIf(any(isinstance(x, unicode) for x in d2.values()))
+        self.failIf(d is d2)  # shouldn't modify in place
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
 
+    @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_keys_only(self):
         d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
         d2 = stringify_dict(d)
         self.assertEqual(d, d2)
-        self.failIf(d is d2) # shouldn't modify in place
-        self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
+        self.failIf(d is d2)  # shouldn't modify in place
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
 
     def test_get_func_args(self):
         def f1(a, b, c):
@@ -194,7 +208,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertEqual(get_func_args(object), [])
 
         # TODO: how do we fix this to return the actual argument names?
-        self.assertEqual(get_func_args(unicode.split), [])
+        self.assertEqual(get_func_args(six.text_type.split), [])
         self.assertEqual(get_func_args(" ".join), [])
         self.assertEqual(get_func_args(operator.itemgetter(2)), [])