Repository: scrapy/scrapy

Issue:
Redirect 308 missing
I did a check on the RedirectMiddleware and noticed that code 308 is missing. Is there a reason for that?
Some websites don't update their sitemap and have a long list of 308 from http to https.

(side note: is there a way to add "s" before a link is scraped?)


Discussion:
Hi @maugch , I don't see any specific reason for that omission, except maybe that the [RFC for HTTP 308](https://tools.ietf.org/html/rfc7538) is rather "new" (2015) and no-one added the code to the middleware.

If would be good that handle 308 indeed.
Interestingly enough, HTTP 308 is tested [in HTTP cache extension](https://github.com/scrapy/scrapy/blob/dd5ab6c5fe54b3d64cdbf89fcfa2b333cd90f22f/scrapy/extensions/httpcache.py#L90).

Resolving patch:
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -64,7 +64,7 @@ def process_response(self, request, response, spider):
                 request.meta.get('handle_httpstatus_all', False)):
             return response
 
-        allowed_status = (301, 302, 303, 307)
+        allowed_status = (301, 302, 303, 307, 308)
         if 'Location' not in response.headers or response.status not in allowed_status:
             return response
 
@@ -72,7 +72,7 @@ def process_response(self, request, response, spider):
 
         redirected_url = urljoin(request.url, location)
 
-        if response.status in (301, 307) or request.method == 'HEAD':
+        if response.status in (301, 307, 308) or request.method == 'HEAD':
             redirected = request.replace(url=redirected_url)
             return self._redirect(redirected, request, spider, response.status)
 


Test patch:
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -322,6 +322,7 @@ def test_response_cacheability(self):
             (True, 203, {'Last-Modified': self.yesterday}),
             (True, 300, {'Last-Modified': self.yesterday}),
             (True, 301, {'Last-Modified': self.yesterday}),
+            (True, 308, {'Last-Modified': self.yesterday}),
             (True, 401, {'Last-Modified': self.yesterday}),
             (True, 404, {'Cache-Control': 'public, max-age=600'}),
             (True, 302, {'Expires': self.tomorrow}),
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -22,12 +22,12 @@ def test_priority_adjust(self):
         req2 = self.mw.process_response(req, rsp, self.spider)
         assert req2.priority > req.priority
 
-    def test_redirect_301(self):
-        def _test(method):
-            url = 'http://www.example.com/301'
+    def test_redirect_3xx_permanent(self):
+        def _test(method, status=301):
+            url = 'http://www.example.com/{}'.format(status)
             url2 = 'http://www.example.com/redirected'
             req = Request(url, method=method)
-            rsp = Response(url, headers={'Location': url2}, status=301)
+            rsp = Response(url, headers={'Location': url2}, status=status)
 
             req2 = self.mw.process_response(req, rsp, self.spider)
             assert isinstance(req2, Request)
@@ -42,6 +42,14 @@ def _test(method):
         _test('POST')
         _test('HEAD')
 
+        _test('GET', status=307)
+        _test('POST', status=307)
+        _test('HEAD', status=307)
+
+        _test('GET', status=308)
+        _test('POST', status=308)
+        _test('HEAD', status=308)
+
     def test_dont_redirect(self):
         url = 'http://www.example.com/301'
         url2 = 'http://www.example.com/redirected'