Repository: scrapy/scrapy

Issue:
Response.follow() method not consistent with Request.__init__(). Missing `flags`.
### Versions Affected:

All versions after PR #2082 when flags were added to `Request`. (>1.4.0)

### Response.follow() method not consistent with Request.__init__(). Missing `flags`.

No sure how this got missed since PR #2082 was merged, but looking at the parameters for `Request.__init__` and `Response.follow` it appears `flags` wasn't added to `Response.follow` to keep it in line with how creating new `Request` instances works. I'm not sure if this was just overlooked or was intentional, when `flags` support was added to requests. It's also missing for the subclasses of `Response` as well (`TextResponse` will need updated, the rest currently use inheritance).

I was looking to be able to add custom `flags` to certain `Response.follow()` calls but realized looking at the source they wouldn't be carried over to the new responses, but this can easily be worked around by manually added the flag after Request creation. This seems like an easy fix to include in 1.8.1 or later but doesn't seem too high priority and I just thought you guys should know or that it should at least be noted somewhere on this repo since the [docs](https://docs.scrapy.org/en/latest/topics/request-response.html#scrapy.http.Response.follow) still say, "It accepts the same arguments as `Request.__init__` method...".



Discussion:
You're right! Thanks for the report. I think this is a reasonable thing, and a good implementation should not take long to get accepted. Do you plan to submit a PR?
If not done, I would like to submit a PR for this. 
As far as I understand, the "flags=None" parameter needs to be added to requests.follow(), right?
Yep, both `Response.follow` and `TextResponse.follow` since it looks like the only subclass that overrides that method. Sorry haven't had time to write a PR for it this weekend, been too busy to even touch my laptop.

Resolving patch:
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -107,7 +107,7 @@ def xpath(self, *a, **kw):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding='utf-8', priority=0,
-               dont_filter=False, errback=None, cb_kwargs=None):
+               dont_filter=False, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -124,6 +124,7 @@ def follow(self, url, callback=None, method='GET', headers=None, body=None,
         elif url is None:
             raise ValueError("url can't be None")
         url = self.urljoin(url)
+
         return Request(
             url=url,
             callback=callback,
@@ -137,11 +138,12 @@ def follow(self, url, callback=None, method='GET', headers=None, body=None,
             dont_filter=dont_filter,
             errback=errback,
             cb_kwargs=cb_kwargs,
+            flags=flags,
         )
 
     def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
                    cookies=None, meta=None, encoding='utf-8', priority=0,
-                   dont_filter=False, errback=None, cb_kwargs=None):
+                   dont_filter=False, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Generator[Request, None, None]
         """
         Return an iterable of :class:`~.Request` instances to follow all links
@@ -169,6 +171,7 @@ def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
                 dont_filter=dont_filter,
                 errback=errback,
                 cb_kwargs=cb_kwargs,
+                flags=flags,
             )
             for url in urls
         )
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -121,7 +121,7 @@ def css(self, query):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding=None, priority=0,
-               dont_filter=False, errback=None, cb_kwargs=None):
+               dont_filter=False, errback=None, cb_kwargs=None, flags=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -157,11 +157,12 @@ def follow(self, url, callback=None, method='GET', headers=None, body=None,
             dont_filter=dont_filter,
             errback=errback,
             cb_kwargs=cb_kwargs,
+            flags=flags,
         )
 
     def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
                    cookies=None, meta=None, encoding=None, priority=0,
-                   dont_filter=False, errback=None, cb_kwargs=None,
+                   dont_filter=False, errback=None, cb_kwargs=None, flags=None,
                    css=None, xpath=None):
         # type: (...) -> Generator[Request, None, None]
         """
@@ -214,6 +215,7 @@ def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=
             dont_filter=dont_filter,
             errback=errback,
             cb_kwargs=cb_kwargs,
+            flags=flags,
         )
 
 


Test patch:
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -166,6 +166,10 @@ def test_follow_whitespace_url(self):
     def test_follow_whitespace_link(self):
         self._assert_followed_url(Link('http://example.com/foo '),
                                   'http://example.com/foo%20')
+    def test_follow_flags(self):
+        res = self.response_class('http://example.com/')
+        fol = res.follow('http://example.com/', flags=['cached', 'allowed'])
+        self.assertEqual(fol.flags, ['cached', 'allowed'])
 
     # Response.follow_all
 
@@ -232,6 +236,17 @@ def test_follow_all_whitespace_links(self):
         expected = [u.replace(' ', '%20') for u in absolute]
         self._assert_followed_all_urls(links, expected)
 
+    def test_follow_all_flags(self):
+        re = self.response_class('http://www.example.com/')
+        urls = [
+            'http://www.example.com/',
+            'http://www.example.com/2',
+            'http://www.example.com/foo',
+        ]
+        fol = re.follow_all(urls, flags=['cached', 'allowed'])
+        for req in fol:
+            self.assertEqual(req.flags, ['cached', 'allowed'])
+
     def _assert_followed_url(self, follow_obj, target_url, response=None):
         if response is None:
             response = self._links_response()
@@ -562,6 +577,22 @@ def test_follow_encoding(self):
         )
         self.assertEqual(req.encoding, 'cp1251')
 
+    def test_follow_flags(self):
+        res = self.response_class('http://example.com/')
+        fol = res.follow('http://example.com/', flags=['cached', 'allowed'])
+        self.assertEqual(fol.flags, ['cached', 'allowed'])
+
+    def test_follow_all_flags(self):
+        re = self.response_class('http://www.example.com/')
+        urls = [
+            'http://www.example.com/',
+            'http://www.example.com/2',
+            'http://www.example.com/foo',
+        ]
+        fol = re.follow_all(urls, flags=['cached', 'allowed'])
+        for req in fol:
+            self.assertEqual(req.flags, ['cached', 'allowed'])
+
     def test_follow_all_css(self):
         expected = [
             'http://example.com/sample3.html',