Repository: scrapy/scrapy
Pull request #2322 (2016-10-12 11:02:11)

Do not interpret non-ASCII bytes in "Location" and percent-encode them
Fixes GH-2321

The idea is to not guess the encoding of "Location" header value
and simply percent-encode non-ASCII bytes,
which should then be re-interpreted correctly by the remote website
in whatever encoding was used originally.

See https://tools.ietf.org/html/rfc3987#section-3.2

This is similar to the changes to safe_url_string in
https://github.com/scrapy/w3lib/pull/45


Patch:
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 6b43fe258..8c7aa361f 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -106,9 +106,12 @@ format::
 
    class JsonWriterPipeline(object):
 
-       def __init__(self):
+       def open_spider(self, spider):
            self.file = open('items.jl', 'wb')
 
+       def close_spider(self, spider):
+           self.file.close()
+
        def process_item(self, item, spider):
            line = json.dumps(dict(item)) + "\n"
            self.file.write(line)
@@ -126,14 +129,7 @@ MongoDB address and database name are specified in Scrapy settings;
 MongoDB collection is named after item class.
 
 The main point of this example is to show how to use :meth:`from_crawler`
-method and how to clean up the resources properly.
-
-.. note::
-
-    Previous example (JsonWriterPipeline) doesn't clean up resources properly.
-    Fixing it is left as an exercise for the reader.
-
-::
+method and how to clean up the resources properly.::
 
     import pymongo
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 0e473709a..af676fccd 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -72,6 +72,8 @@ scrapy.Spider
        spider that crawls ``mywebsite.com`` would often be called
        ``mywebsite``.
 
+       .. note:: In Python 2 this must be ASCII only.
+
    .. attribute:: allowed_domains
 
        An optional list of strings containing domains that this spider is
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index b643d935b..4bd4c6166 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -13,7 +13,7 @@ try:
     from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.web.iweb import IPolicyForHTTPS
 
-    from scrapy.core.downloader.tls import ScrapyClientTLSOptions
+    from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
 
 
     @implementer(IPolicyForHTTPS)
@@ -46,7 +46,9 @@ try:
             #   not calling super(..., self).__init__
             return CertificateOptions(verify=False,
                         method=getattr(self, 'method',
-                                       getattr(self, '_ssl_method', None)))
+                                       getattr(self, '_ssl_method', None)),
+                        fixBrokenPeers=True,
+                        acceptableCiphers=DEFAULT_CIPHERS)
 
         # kept for old-style HTTP/1.0 downloader context twisted calls,
         # e.g. connectSSL()
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 00c94ee2e..955b7630c 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -28,6 +28,7 @@ try:
         SSL_CB_HANDSHAKE_START = 0x10
         SSL_CB_HANDSHAKE_DONE = 0x20
 
+    from twisted.internet.ssl import AcceptableCiphers
     from twisted.internet._sslverify import (ClientTLSOptions,
                                              _maybeSetHostNameIndication,
                                              verifyHostname,
@@ -60,6 +61,8 @@ try:
                         'from host "{}" (exception: {})'.format(
                             self._hostnameASCII, repr(e)))
 
+    DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
+
 except ImportError:
     # ImportError should not matter for older Twisted versions
     # as the above is not used in the fallback ScrapyClientContextFactory
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 4ed7e4c24..db276eefb 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,9 +1,10 @@
 import logging
 from six.moves.urllib.parse import urljoin
 
+from w3lib.url import safe_url_string
+
 from scrapy.http import HtmlResponse
 from scrapy.utils.response import get_meta_refresh
-from scrapy.utils.python import to_native_str
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 
 logger = logging.getLogger(__name__)
@@ -65,8 +66,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
         if 'Location' not in response.headers or response.status not in allowed_status:
             return response
 
-        # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8
-        location = to_native_str(response.headers['location'].decode('latin1'))
+        location = safe_url_string(response.headers['location'])
 
         redirected_url = urljoin(request.url, location)
 
diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl
index 2d3b96e38..99e5d43b2 100644
--- a/scrapy/templates/spiders/basic.tmpl
+++ b/scrapy/templates/spiders/basic.tmpl
@@ -5,9 +5,7 @@ import scrapy
 class $classname(scrapy.Spider):
     name = "$name"
     allowed_domains = ["$domain"]
-    start_urls = (
-        'http://www.$domain/',
-    )
+    start_urls = ['http://$domain/']
 
     def parse(self, response):
         pass
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 154237d9c..802cb88a1 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -7,7 +7,7 @@ from scrapy.spiders import CrawlSpider, Rule
 class $classname(CrawlSpider):
     name = '$name'
     allowed_domains = ['$domain']
-    start_urls = ['http://www.$domain/']
+    start_urls = ['http://$domain/']
 
     rules = (
         Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl
index 0544e0ae7..c2e4bacfe 100644
--- a/scrapy/templates/spiders/csvfeed.tmpl
+++ b/scrapy/templates/spiders/csvfeed.tmpl
@@ -5,7 +5,7 @@ from scrapy.spiders import CSVFeedSpider
 class $classname(CSVFeedSpider):
     name = '$name'
     allowed_domains = ['$domain']
-    start_urls = ['http://www.$domain/feed.csv']
+    start_urls = ['http://$domain/feed.csv']
     # headers = ['id', 'name', 'description', 'image_link']
     # delimiter = '\t'
 
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index d8ff61f6e..7c2ff8850 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -5,7 +5,7 @@ from scrapy.spiders import XMLFeedSpider
 class $classname(XMLFeedSpider):
     name = '$name'
     allowed_domains = ['$domain']
-    start_urls = ['http://www.$domain/feed.xml']
+    start_urls = ['http://$domain/feed.xml']
     iterator = 'iternodes' # you can change this; see the docs
     itertag = 'item' # change it accordingly
 


Test patch:
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 9db073cc5..e8c92affa 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -157,15 +157,15 @@ class RedirectMiddlewareTest(unittest.TestCase):
         latin1_location = u'/ação'.encode('latin1')  # HTTP historically supports latin1
         resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
-        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
+        perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
         self.assertEquals(perc_encoded_utf8_url, req_result.url)
 
-    def test_location_with_wrong_encoding(self):
+    def test_utf8_location(self):
         req = Request('http://scrapytest.org/first')
-        utf8_location = u'/ação'  # header with wrong encoding (utf-8)
+        utf8_location = u'/ação'.encode('utf-8')  # header using UTF-8 encoding
         resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
-        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%83%C2%A7%C3%83%C2%A3o'
+        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
         self.assertEquals(perc_encoded_utf8_url, req_result.url)