Repository: scrapy/scrapy
Pull request #1730 (2016-01-26 20:28:33)

[MRG+1] response.text
Fixes GH-1729.

TextResponse.body_as_unicode() is old and common, so it is not deprecated. I think it is better not to flood users with warnings without a compelling reason.


Patch:
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index ea64d1599..2e92961a9 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -445,10 +445,10 @@ Response objects
 
     .. attribute:: Response.body
 
-        A str containing the body of this Response. Keep in mind that Response.body
-        is always a str. If you want the unicode version use
-        :meth:`TextResponse.body_as_unicode` (only available in
-        :class:`TextResponse` and subclasses).
+        The body of this Response. Keep in mind that Response.body
+        is always a bytes object. If you want the unicode version use
+        :attr:`TextResponse.txt` (only available in :class:`TextResponse`
+        and subclasses).
 
         This attribute is read-only. To change the body of a Response use
         :meth:`replace`.
@@ -542,6 +542,21 @@ TextResponse objects
     :class:`TextResponse` objects support the following attributes in addition
     to the standard :class:`Response` ones:
 
+    .. attribute:: TextResponse.text
+
+       Response body, as unicode.
+
+       The same as ``response.body.decode(response.encoding)``, but the
+       result is cached after the first call, so you can access
+       ``response.text`` multiple times without extra overhead.
+
+       .. note::
+
+            ``unicode(response.body)`` is not a correct way to convert response
+            body to unicode: you would be using the system default encoding
+            (typically `ascii`) instead of the response encoding.
+
+
     .. attribute:: TextResponse.encoding
 
        A string with the encoding of this response. The encoding is resolved by
@@ -568,20 +583,6 @@ TextResponse objects
     :class:`TextResponse` objects support the following methods in addition to
     the standard :class:`Response` ones:
 
-    .. method:: TextResponse.body_as_unicode()
-
-        Returns the body of the response as unicode. This is equivalent to::
-
-            response.body.decode(response.encoding)
-
-        But **not** equivalent to::
-
-            unicode(response.body)
-
-        Since, in the latter case, you would be using the system default encoding
-        (typically `ascii`) to convert the body to unicode, instead of the response
-        encoding.
-
     .. method:: TextResponse.xpath(query)
 
         A shortcut to ``TextResponse.selector.xpath(query)``::
@@ -594,6 +595,11 @@ TextResponse objects
 
             response.css('p')
 
+    .. method:: TextResponse.body_as_unicode()
+
+        The same as :attr:`text`, but available as a method. This method is
+        kept for backwards compatibility; please prefer ``response.text``.
+
 
 HtmlResponse objects
 --------------------
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 6b543b823..da373eca2 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -63,7 +63,7 @@ class AjaxCrawlMiddleware(object):
         Return True if a page without hash fragment could be "AJAX crawlable"
         according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
         """
-        body = response.body_as_unicode()[:self.lookup_bytes]
+        body = response.text[:self.lookup_bytes]
         return _has_ajaxcrawlable_meta(body)
 
 
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index c061c2407..d4a33dc36 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -83,8 +83,8 @@ class RobotsTxtMiddleware(object):
     def _parse_robots(self, response, netloc):
         rp = robotparser.RobotFileParser(response.url)
         body = ''
-        if hasattr(response, 'body_as_unicode'):
-            body = response.body_as_unicode()
+        if hasattr(response, 'text'):
+            body = response.text
         else: # last effort try
             try:
                 body = response.body.decode('utf-8')
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 5501634d3..2862dc096 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -64,8 +64,8 @@ def _urlencode(seq, enc):
 
 def _get_form(response, formname, formid, formnumber, formxpath):
     """Find the form element """
-    text = response.body_as_unicode()
-    root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response))
+    root = create_root_node(response.text, lxml.html.HTMLParser,
+                            base_url=get_base_url(response))
     forms = root.xpath('//form')
     if not forms:
         raise ValueError("No <form> element found in %s" % response)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 1c416bf82..9c667ab7e 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -67,6 +67,11 @@ class TextResponse(Response):
             self._cached_ubody = html_to_unicode(charset, self.body)[1]
         return self._cached_ubody
 
+    @property
+    def text(self):
+        """ Body as unicode """
+        return self.body_as_unicode()
+
     def urljoin(self, url):
         """Join this Response's url with a possible relative url to form an
         absolute interpretation of the latter."""
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 5d77f7624..15f3d26df 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -60,7 +60,7 @@ class Selector(_ParselSelector, object_ref):
             response = _response_from_text(text, st)
 
         if response is not None:
-            text = response.body_as_unicode()
+            text = response.text
             kwargs.setdefault('base_url', response.url)
 
         self.response = response
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index b0688791e..73857b410 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):
         if not unicode:
             return obj.body
         elif isinstance(obj, TextResponse):
-            return obj.body_as_unicode()
+            return obj.text
         else:
             return obj.body.decode('utf-8')
     elif isinstance(obj, six.text_type):
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index c4ad52f14..73db2641e 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -25,7 +25,7 @@ _baseurl_cache = weakref.WeakKeyDictionary()
 def get_base_url(response):
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
-        text = response.body_as_unicode()[0:4096]
+        text = response.text[0:4096]
         _baseurl_cache[response] = html.get_base_url(text, response.url,
             response.encoding)
     return _baseurl_cache[response]
@@ -37,7 +37,7 @@ _metaref_cache = weakref.WeakKeyDictionary()
 def get_meta_refresh(response):
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
-        text = response.body_as_unicode()[0:4096]
+        text = response.text[0:4096]
         text = _noscript_re.sub(u'', text)
         text = _script_re.sub(u'', text)
         _metaref_cache[response] = html.get_meta_refresh(text, response.url,


Test patch:
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 9f2c02bff..baf6ef1bf 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -55,12 +55,11 @@ class TestSpider(Spider):
 
     def parse_item(self, response):
         item = self.item_cls()
-        body = response.body_as_unicode()
-        m = self.name_re.search(body)
+        m = self.name_re.search(response.text)
         if m:
             item['name'] = m.group(1)
         item['url'] = response.url
-        m = self.price_re.search(body)
+        m = self.price_re.search(response.text)
         if m:
             item['price'] = m.group(1)
         return item
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 710a5b29d..c7f36687a 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -107,9 +107,11 @@ class BaseResponseTest(unittest.TestCase):
             body_bytes = body
 
         assert isinstance(response.body, bytes)
+        assert isinstance(response.text, six.text_type)
         self._assert_response_encoding(response, encoding)
         self.assertEqual(response.body, body_bytes)
         self.assertEqual(response.body_as_unicode(), body_unicode)
+        self.assertEqual(response.text, body_unicode)
 
     def _assert_response_encoding(self, response, encoding):
         self.assertEqual(response.encoding, resolve_encoding(encoding))
@@ -171,6 +173,10 @@ class TextResponseTest(BaseResponseTest):
         self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type))
         self.assertEqual(r1.body_as_unicode(), unicode_string)
 
+        # check response.text
+        self.assertTrue(isinstance(r1.text, six.text_type))
+        self.assertEqual(r1.text, unicode_string)
+
     def test_encoding(self):
         r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3")
         r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
@@ -219,12 +225,12 @@ class TextResponseTest(BaseResponseTest):
                                  headers={"Content-type": ["text/html; charset=utf-8"]},
                                  body=b"\xef\xbb\xbfWORD\xe3\xab")
         self.assertEqual(r6.encoding, 'utf-8')
-        self.assertEqual(r6.body_as_unicode(), u'WORD\ufffd\ufffd')
+        self.assertEqual(r6.text, u'WORD\ufffd\ufffd')
 
     def test_bom_is_removed_from_body(self):
         # Inferring encoding from body also cache decoded body as sideeffect,
         # this test tries to ensure that calling response.encoding and
-        # response.body_as_unicode() in indistint order doesn't affect final
+        # response.text in indistint order doesn't affect final
         # values for encoding and decoded body.
         url = 'http://example.com'
         body = b"\xef\xbb\xbfWORD"
@@ -233,9 +239,9 @@ class TextResponseTest(BaseResponseTest):
         # Test response without content-type and BOM encoding
         response = self.response_class(url, body=body)
         self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.body_as_unicode(), u'WORD')
+        self.assertEqual(response.text, u'WORD')
         response = self.response_class(url, body=body)
-        self.assertEqual(response.body_as_unicode(), u'WORD')
+        self.assertEqual(response.text, u'WORD')
         self.assertEqual(response.encoding, 'utf-8')
 
         # Body caching sideeffect isn't triggered when encoding is declared in
@@ -243,9 +249,9 @@ class TextResponseTest(BaseResponseTest):
         # body
         response = self.response_class(url, headers=headers, body=body)
         self.assertEqual(response.encoding, 'utf-8')
-        self.assertEqual(response.body_as_unicode(), u'WORD')
+        self.assertEqual(response.text, u'WORD')
         response = self.response_class(url, headers=headers, body=body)
-        self.assertEqual(response.body_as_unicode(), u'WORD')
+        self.assertEqual(response.text, u'WORD')
         self.assertEqual(response.encoding, 'utf-8')
 
     def test_replace_wrong_encoding(self):
@@ -253,18 +259,18 @@ class TextResponseTest(BaseResponseTest):
         r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
         # XXX: Policy for replacing invalid chars may suffer minor variations
         # but it should always contain the unicode replacement char (u'\ufffd')
-        assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())
-        assert u'PREFIX' in r.body_as_unicode(), repr(r.body_as_unicode())
-        assert u'SUFFIX' in r.body_as_unicode(), repr(r.body_as_unicode())
+        assert u'\ufffd' in r.text, repr(r.text)
+        assert u'PREFIX' in r.text, repr(r.text)
+        assert u'SUFFIX' in r.text, repr(r.text)
 
         # Do not destroy html tags due to encoding bugs
         r = self.response_class("http://example.com", encoding='utf-8', \
                 body=b'\xf0<span>value</span>')
-        assert u'<span>value</span>' in r.body_as_unicode(), repr(r.body_as_unicode())
+        assert u'<span>value</span>' in r.text, repr(r.text)
 
         # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
-        #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX')
-        #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())
+        #r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
+        #assert u'\ufffd' in r.text, repr(r.text)
 
     def test_selector(self):
         body = b"<html><head><title>Some page</title><body></body></html>"