Repository: scrapy/scrapy

Issue:
Scrapy uses `file://` URL as referer
If I use a local file in a request (e.g. 

``` python
def start_requests(self):
    yield Request(path_to_file_uri(filename))

def parse(self, response):
    for url in json.loads(response.text):
        yield Request(url)
```

and then make a HTTP request in the callback, scrapy uses the file URL as the `referer` in the HTTP header. 

I think `file://` URL should be avoided from being used as the `referer` value (or generally any protocol other than http/s) because
1. it is not part of the normal hyperlink navigation
2. it exposes the local file structure to the remote site



Discussion:
+1, makes sense. 

It seems there are more rules for Referer header which browsers implement. I haven't checked it myself, but
http://webmasters.stackexchange.com/questions/47405/how-can-i-pass-referrer-header-from-my-https-domain-to-http-domains suggests that Referer is not sent for requests sent from https pages to any pages not in the same domain. RFC doesn't require browsers to do so, but it looks they do it for security/privacy reasons. 

http://www.redpill-linpro.com/sysadvent//2015/12/17/referer-header.html gives another list of rules.


Resolving patch:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -234,6 +234,7 @@
 REDIRECT_PRIORITY_ADJUST = +2
 
 REFERER_ENABLED = True
+REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
 
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -2,22 +2,355 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
+from six.moves.urllib.parse import urlparse
+import warnings
 
-from scrapy.http import Request
+from w3lib.url import safe_url_string
+
+from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
+from scrapy import signals
+from scrapy.utils.python import to_native_str
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.misc import load_object
+from scrapy.utils.url import strip_url
+
+
+LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
+
+POLICY_NO_REFERRER = "no-referrer"
+POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade"
+POLICY_SAME_ORIGIN = "same-origin"
+POLICY_ORIGIN = "origin"
+POLICY_STRICT_ORIGIN = "strict-origin"
+POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin"
+POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin"
+POLICY_UNSAFE_URL = "unsafe-url"
+POLICY_SCRAPY_DEFAULT = "scrapy-default"
+
+
+class ReferrerPolicy(object):
+
+    NOREFERRER_SCHEMES = LOCAL_SCHEMES
+
+    def referrer(self, response_url, request_url):
+        raise NotImplementedError()
+
+    def stripped_referrer(self, url):
+        if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
+            return self.strip_url(url)
+
+    def origin_referrer(self, url):
+        if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
+            return self.origin(url)
+
+    def strip_url(self, url, origin_only=False):
+        """
+        https://www.w3.org/TR/referrer-policy/#strip-url
+
+        If url is null, return no referrer.
+        If url's scheme is a local scheme, then return no referrer.
+        Set url's username to the empty string.
+        Set url's password to null.
+        Set url's fragment to null.
+        If the origin-only flag is true, then:
+            Set url's path to null.
+            Set url's query to null.
+        Return url.
+        """
+        if not url:
+            return None
+        return strip_url(url,
+                         strip_credentials=True,
+                         strip_fragment=True,
+                         strip_default_port=True,
+                         origin_only=origin_only)
+
+    def origin(self, url):
+        """Return serialized origin (scheme, host, path) for a request or response URL."""
+        return self.strip_url(url, origin_only=True)
+
+    def potentially_trustworthy(self, url):
+        # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy
+        parsed_url = urlparse(url)
+        if parsed_url.scheme in ('data',):
+            return False
+        return self.tls_protected(url)
+
+    def tls_protected(self, url):
+        return urlparse(url).scheme in ('https', 'ftps')
+
+
+class NoReferrerPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer
+
+    The simplest policy is "no-referrer", which specifies that no referrer information
+    is to be sent along with requests made from a particular request client to any origin.
+    The header will be omitted entirely.
+    """
+    name = POLICY_NO_REFERRER
+
+    def referrer(self, response_url, request_url):
+        return None
+
+
+class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
+
+    The "no-referrer-when-downgrade" policy sends a full URL along with requests
+    from a TLS-protected environment settings object to a potentially trustworthy URL,
+    and requests from clients which are not TLS-protected to any origin.
+
+    Requests from TLS-protected clients to non-potentially trustworthy URLs,
+    on the other hand, will contain no referrer information.
+    A Referer HTTP header will not be sent.
+
+    This is a user agent's default behavior, if no policy is otherwise specified.
+    """
+    name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
+
+    def referrer(self, response_url, request_url):
+        if not self.tls_protected(response_url) or self.tls_protected(request_url):
+            return self.stripped_referrer(response_url)
+
+
+class SameOriginPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin
+
+    The "same-origin" policy specifies that a full URL, stripped for use as a referrer,
+    is sent as referrer information when making same-origin requests from a particular request client.
+
+    Cross-origin requests, on the other hand, will contain no referrer information.
+    A Referer HTTP header will not be sent.
+    """
+    name = POLICY_SAME_ORIGIN
+
+    def referrer(self, response_url, request_url):
+        if self.origin(response_url) == self.origin(request_url):
+            return self.stripped_referrer(response_url)
+
+
+class OriginPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-origin
+
+    The "origin" policy specifies that only the ASCII serialization
+    of the origin of the request client is sent as referrer information
+    when making both same-origin requests and cross-origin requests
+    from a particular request client.
+    """
+    name = POLICY_ORIGIN
+
+    def referrer(self, response_url, request_url):
+        return self.origin_referrer(response_url)
+
+
+class StrictOriginPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin
+
+    The "strict-origin" policy sends the ASCII serialization
+    of the origin of the request client when making requests:
+    - from a TLS-protected environment settings object to a potentially trustworthy URL, and
+    - from non-TLS-protected environment settings objects to any origin.
+
+    Requests from TLS-protected request clients to non- potentially trustworthy URLs,
+    on the other hand, will contain no referrer information.
+    A Referer HTTP header will not be sent.
+    """
+    name = POLICY_STRICT_ORIGIN
+
+    def referrer(self, response_url, request_url):
+        if ((self.tls_protected(response_url) and
+             self.potentially_trustworthy(request_url))
+            or not self.tls_protected(response_url)):
+            return self.origin_referrer(response_url)
+
+
+class OriginWhenCrossOriginPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin
+
+    The "origin-when-cross-origin" policy specifies that a full URL,
+    stripped for use as a referrer, is sent as referrer information
+    when making same-origin requests from a particular request client,
+    and only the ASCII serialization of the origin of the request client
+    is sent as referrer information when making cross-origin requests
+    from a particular request client.
+    """
+    name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
+
+    def referrer(self, response_url, request_url):
+        origin = self.origin(response_url)
+        if origin == self.origin(request_url):
+            return self.stripped_referrer(response_url)
+        else:
+            return origin
+
+
+class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
+
+    The "strict-origin-when-cross-origin" policy specifies that a full URL,
+    stripped for use as a referrer, is sent as referrer information
+    when making same-origin requests from a particular request client,
+    and only the ASCII serialization of the origin of the request client
+    when making cross-origin requests:
+
+    - from a TLS-protected environment settings object to a potentially trustworthy URL, and
+    - from non-TLS-protected environment settings objects to any origin.
+
+    Requests from TLS-protected clients to non- potentially trustworthy URLs,
+    on the other hand, will contain no referrer information.
+    A Referer HTTP header will not be sent.
+    """
+    name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
+
+    def referrer(self, response_url, request_url):
+        origin = self.origin(response_url)
+        if origin == self.origin(request_url):
+            return self.stripped_referrer(response_url)
+        elif ((self.tls_protected(response_url) and
+               self.potentially_trustworthy(request_url))
+              or not self.tls_protected(response_url)):
+            return self.origin_referrer(response_url)
+
+
+class UnsafeUrlPolicy(ReferrerPolicy):
+    """
+    https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url
+
+    The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer,
+    is sent along with both cross-origin requests
+    and same-origin requests made from a particular request client.
+
+    Note: The policy's name doesn't lie; it is unsafe.
+    This policy will leak origins and paths from TLS-protected resources
+    to insecure origins.
+    Carefully consider the impact of setting such a policy for potentially sensitive documents.
+    """
+    name = POLICY_UNSAFE_URL
+
+    def referrer(self, response_url, request_url):
+        return self.stripped_referrer(response_url)
+
+
+class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
+    """
+    A variant of "no-referrer-when-downgrade",
+    with the addition that "Referer" is not sent if the parent request was
+    using ``file://`` or ``s3://`` scheme.
+    """
+    NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
+    name = POLICY_SCRAPY_DEFAULT
+
+
+_policy_classes = {p.name: p for p in (
+    NoReferrerPolicy,
+    NoReferrerWhenDowngradePolicy,
+    SameOriginPolicy,
+    OriginPolicy,
+    StrictOriginPolicy,
+    OriginWhenCrossOriginPolicy,
+    StrictOriginWhenCrossOriginPolicy,
+    UnsafeUrlPolicy,
+    DefaultReferrerPolicy,
+)}
+
+
+def _load_policy_class(policy, warning_only=False):
+    """
+    Expect a string for the path to the policy class,
+    otherwise try to interpret the string as a standard value
+    from https://www.w3.org/TR/referrer-policy/#referrer-policies
+    """
+    try:
+        return load_object(policy)
+    except ValueError:
+        try:
+            return _policy_classes[policy.lower()]
+        except KeyError:
+            msg = "Could not load referrer policy %r" % policy
+            if not warning_only:
+                raise RuntimeError(msg)
+            else:
+                warnings.warn(msg, RuntimeWarning)
+                return None
+
 
 class RefererMiddleware(object):
 
+    def __init__(self, settings=None):
+        self.default_policy = DefaultReferrerPolicy
+        if settings is not None:
+            self.default_policy = _load_policy_class(
+                settings.get('REFERRER_POLICY'))
+
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('REFERER_ENABLED'):
             raise NotConfigured
-        return cls()
+        mw = cls(crawler.settings)
+
+        # Note: this hook is a bit of a hack to intercept redirections
+        crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled)
+
+        return mw
+
+    def policy(self, resp_or_url, request):
+        """
+        Determine Referrer-Policy to use from a parent Response (or URL),
+        and a Request to be sent.
+
+        - if a valid policy is set in Request meta, it is used.
+        - if the policy is set in meta but is wrong (e.g. a typo error),
+          the policy from settings is used
+        - if the policy is not set in Request meta,
+          but there is a Referrer-policy header in the parent response,
+          it is used if valid
+        - otherwise, the policy from settings is used.
+        """
+        policy_name = request.meta.get('referrer_policy')
+        if policy_name is None:
+            if isinstance(resp_or_url, Response):
+                policy_name = to_native_str(
+                    resp_or_url.headers.get('Referrer-Policy', '').decode('latin1'))
+        if policy_name is None:
+            return self.default_policy()
+
+        cls = _load_policy_class(policy_name, warning_only=True)
+        return cls() if cls else self.default_policy()
 
     def process_spider_output(self, response, result, spider):
         def _set_referer(r):
             if isinstance(r, Request):
-                r.headers.setdefault('Referer', response.url)
+                referrer = self.policy(response, r).referrer(response.url, r.url)
+                if referrer is not None:
+                    r.headers.setdefault('Referer', referrer)
             return r
         return (_set_referer(r) for r in result or ())
 
+    def request_scheduled(self, request, spider):
+        # check redirected request to patch "Referer" header if necessary
+        redirected_urls = request.meta.get('redirect_urls', [])
+        if redirected_urls:
+            request_referrer = request.headers.get('Referer')
+            # we don't patch the referrer value if there is none
+            if request_referrer is not None:
+                # the request's referrer header value acts as a surrogate
+                # for the parent response URL
+                #
+                # Note: if the 3xx response contained a Referrer-Policy header,
+                #       the information is not available using this hook
+                parent_url = safe_url_string(request_referrer)
+                policy_referrer = self.policy(parent_url, request).referrer(
+                    parent_url, request.url)
+                if policy_referrer != request_referrer:
+                    if policy_referrer is None:
+                        request.headers.pop('Referer')
+                    else:
+                        request.headers['Referer'] = policy_referrer
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -7,7 +7,7 @@
 """
 import posixpath
 import re
-from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse)
+from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse)
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
 # move doesn't break old code
@@ -103,3 +103,34 @@ def guess_scheme(url):
         return any_to_uri(url)
     else:
         return add_http_if_no_scheme(url)
+
+
+def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):
+
+    """Strip URL string from some of its components:
+
+    - `strip_credentials` removes "user:password@"
+    - `strip_default_port` removes ":80" (resp. ":443", ":21")
+      from http:// (resp. https://, ftp://) URLs
+    - `origin_only` replaces path component with "/", also dropping
+      query and fragment components ; it also strips credentials
+    - `strip_fragment` drops any #fragment component
+    """
+
+    parsed_url = urlparse(url)
+    netloc = parsed_url.netloc
+    if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password):
+        netloc = netloc.split('@')[-1]
+    if strip_default_port and parsed_url.port:
+        if (parsed_url.scheme, parsed_url.port) in (('http', 80),
+                                                    ('https', 443),
+                                                    ('ftp', 21)):
+            netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '')
+    return urlunparse((
+        parsed_url.scheme,
+        netloc,
+        '/' if origin_only else parsed_url.path,
+        '' if origin_only else parsed_url.params,
+        '' if origin_only else parsed_url.query,
+        '' if strip_fragment else parsed_url.fragment
+    ))


Test patch:
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,21 +1,867 @@
+from six.moves.urllib.parse import urlparse
 from unittest import TestCase
+import warnings
 
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, Request
+from scrapy.settings import Settings
 from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.referer import RefererMiddleware
+from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
+from scrapy.spidermiddlewares.referer import RefererMiddleware, \
+    POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \
+    POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \
+    POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \
+    POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \
+    DefaultReferrerPolicy, \
+    NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \
+    OriginWhenCrossOriginPolicy, OriginPolicy, \
+    StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \
+    SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy
 
 
 class TestRefererMiddleware(TestCase):
 
+    req_meta = {}
+    resp_headers = {}
+    settings = {}
+    scenarii = [
+        ('http://scrapytest.org', 'http://scrapytest.org/',  b'http://scrapytest.org'),
+    ]
+
     def setUp(self):
         self.spider = Spider('foo')
-        self.mw = RefererMiddleware()
+        settings = Settings(self.settings)
+        self.mw = RefererMiddleware(settings)
+
+    def get_request(self, target):
+        return Request(target, meta=self.req_meta)
+
+    def get_response(self, origin):
+        return Response(origin, headers=self.resp_headers)
+
+    def test(self):
+
+        for origin, target, referrer in self.scenarii:
+            response = self.get_response(origin)
+            request = self.get_request(target)
+            out = list(self.mw.process_spider_output(response, [request], self.spider))
+            self.assertEquals(out[0].headers.get('Referer'), referrer)
+
+
+class MixinDefault(object):
+    """
+    Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
+
+    with some additional filtering of s3://
+    """
+    scenarii = [
+        ('https://example.com/',    'https://scrapy.org/',  b'https://example.com/'),
+        ('http://example.com/',     'http://scrapy.org/',   b'http://example.com/'),
+        ('http://example.com/',     'https://scrapy.org/',  b'http://example.com/'),
+        ('https://example.com/',    'http://scrapy.org/',   None),
+
+        # no credentials leak
+        ('http://user:password@example.com/',  'https://scrapy.org/', b'http://example.com/'),
+
+        # no referrer leak for local schemes
+        ('file:///home/path/to/somefile.html',  'https://scrapy.org/', None),
+        ('file:///home/path/to/somefile.html',  'http://scrapy.org/',  None),
+
+        # no referrer leak for s3 origins
+        ('s3://mybucket/path/to/data.csv',  'https://scrapy.org/', None),
+        ('s3://mybucket/path/to/data.csv',  'http://scrapy.org/',  None),
+    ]
+
+
+class MixinNoReferrer(object):
+    scenarii = [
+        ('https://example.com/page.html',       'https://example.com/', None),
+        ('http://www.example.com/',             'https://scrapy.org/',  None),
+        ('http://www.example.com/',             'http://scrapy.org/',   None),
+        ('https://www.example.com/',            'http://scrapy.org/',   None),
+        ('file:///home/path/to/somefile.html',  'http://scrapy.org/',   None),
+    ]
+
+
+class MixinNoReferrerWhenDowngrade(object):
+    scenarii = [
+        # TLS to TLS: send non-empty referrer
+        ('https://example.com/page.html',       'https://not.example.com/', b'https://example.com/page.html'),
+        ('https://example.com/page.html',       'https://scrapy.org/',      b'https://example.com/page.html'),
+        ('https://example.com:443/page.html',   'https://scrapy.org/',      b'https://example.com/page.html'),
+        ('https://example.com:444/page.html',   'https://scrapy.org/',      b'https://example.com:444/page.html'),
+        ('ftps://example.com/urls.zip',         'https://scrapy.org/',      b'ftps://example.com/urls.zip'),
+
+        # TLS to non-TLS: do not send referrer
+        ('https://example.com/page.html',       'http://not.example.com/',  None),
+        ('https://example.com/page.html',       'http://scrapy.org/',       None),
+        ('ftps://example.com/urls.zip',         'http://scrapy.org/',       None),
+
+        # non-TLS to TLS or non-TLS: send referrer
+        ('http://example.com/page.html',        'https://not.example.com/', b'http://example.com/page.html'),
+        ('http://example.com/page.html',        'https://scrapy.org/',      b'http://example.com/page.html'),
+        ('http://example.com:8080/page.html',   'https://scrapy.org/',      b'http://example.com:8080/page.html'),
+        ('http://example.com:80/page.html',     'http://not.example.com/',  b'http://example.com/page.html'),
+        ('http://example.com/page.html',        'http://scrapy.org/',       b'http://example.com/page.html'),
+        ('http://example.com:443/page.html',    'http://scrapy.org/',       b'http://example.com:443/page.html'),
+        ('ftp://example.com/urls.zip',          'http://scrapy.org/',       b'ftp://example.com/urls.zip'),
+        ('ftp://example.com/urls.zip',          'https://scrapy.org/',      b'ftp://example.com/urls.zip'),
+
+        # test for user/password stripping
+        ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
+    ]
+
+
+class MixinSameOrigin(object):
+    scenarii = [
+        # Same origin (protocol, host, port): send referrer
+        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
+        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+
+        # Different host: do NOT send referrer
+        ('https://example.com/page.html',       'https://not.example.com/otherpage.html',   None),
+        ('http://example.com/page.html',        'http://not.example.com/otherpage.html',    None),
+        ('http://example.com/page.html',        'http://www.example.com/otherpage.html',    None),
+
+        # Different port: do NOT send referrer
+        ('https://example.com:444/page.html',   'https://example.com/not-page.html',    None),
+        ('http://example.com:81/page.html',     'http://example.com/not-page.html',     None),
+        ('http://example.com/page.html',        'http://example.com:81/not-page.html',  None),
+
+        # Different protocols: do NOT send refferer
+        ('https://example.com/page.html',   'http://example.com/not-page.html',     None),
+        ('https://example.com/page.html',   'http://not.example.com/',              None),
+        ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
+        ('ftp://example.com/urls.zip',      'http://example.com/not-page.html',     None),
+        ('ftps://example.com/urls.zip',     'https://example.com/not-page.html',    None),
+
+        # test for user/password stripping
+        ('https://user:password@example.com/page.html', 'https://example.com/not-page.html',    b'https://example.com/page.html'),
+        ('https://user:password@example.com/page.html', 'http://example.com/not-page.html',     None),
+    ]
+
+
+class MixinOrigin(object):
+    scenarii = [
+        # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
+        ('https://example.com/page.html',   'https://example.com/not-page.html',    b'https://example.com/'),
+        ('https://example.com/page.html',   'https://scrapy.org',                   b'https://example.com/'),
+        ('https://example.com/page.html',   'http://scrapy.org',                    b'https://example.com/'),
+        ('http://example.com/page.html',    'http://scrapy.org',                    b'http://example.com/'),
+
+        # test for user/password stripping
+        ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
+    ]
+
+
+class MixinStrictOrigin(object):
+    scenarii = [
+        # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades
+        ('https://example.com/page.html',   'https://example.com/not-page.html',    b'https://example.com/'),
+        ('https://example.com/page.html',   'https://scrapy.org',                   b'https://example.com/'),
+        ('http://example.com/page.html',    'http://scrapy.org',                    b'http://example.com/'),
+
+        # downgrade: send nothing
+        ('https://example.com/page.html',   'http://scrapy.org',                    None),
+
+        # upgrade: send origin
+        ('http://example.com/page.html',    'https://scrapy.org',                   b'http://example.com/'),
+
+        # test for user/password stripping
+        ('https://user:password@example.com/page.html', 'https://scrapy.org',       b'https://example.com/'),
+        ('https://user:password@example.com/page.html', 'http://scrapy.org',        None),
+    ]
+
+
+class MixinOriginWhenCrossOrigin(object):
+    scenarii = [
+        # Same origin (protocol, host, port): send referrer
+        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
+        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+
+        # Different host: send origin as referrer
+        ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
+        ('https://example2.com/page.html',  'https://not.example2.com/otherpage.html',  b'https://example2.com/'),
+        ('http://example2.com/page.html',   'http://not.example2.com/otherpage.html',   b'http://example2.com/'),
+        # exact match required
+        ('http://example2.com/page.html',   'http://www.example2.com/otherpage.html',   b'http://example2.com/'),
+
+        # Different port: send origin as referrer
+        ('https://example3.com:444/page.html',  'https://example3.com/not-page.html',   b'https://example3.com:444/'),
+        ('http://example3.com:81/page.html',    'http://example3.com/not-page.html',    b'http://example3.com:81/'),
+
+        # Different protocols: send origin as referrer
+        ('https://example4.com/page.html',  'http://example4.com/not-page.html',    b'https://example4.com/'),
+        ('https://example4.com/page.html',  'http://not.example4.com/',             b'https://example4.com/'),
+        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+        ('ftp://example4.com/urls.zip',     'http://example4.com/not-page.html',    b'ftp://example4.com/'),
+        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+
+        # test for user/password stripping
+        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
+        # TLS to non-TLS downgrade: send origin
+        ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   b'https://example5.com/'),
+    ]
+
+
+class MixinStrictOriginWhenCrossOrigin(object):
+    scenarii = [
+        # Same origin (protocol, host, port): send referrer
+        ('https://example.com/page.html',       'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('https://example.com:443/page.html',   'https://example.com/not-page.html',        b'https://example.com/page.html'),
+        ('http://example.com:80/page.html',     'http://example.com/not-page.html',         b'http://example.com/page.html'),
+        ('http://example.com/page.html',        'http://example.com:80/not-page.html',      b'http://example.com/page.html'),
+        ('http://example.com:8888/page.html',   'http://example.com:8888/not-page.html',    b'http://example.com:8888/page.html'),
+
+        # Different host: send origin as referrer
+        ('https://example2.com/page.html',  'https://scrapy.org/otherpage.html',        b'https://example2.com/'),
+        ('https://example2.com/page.html',  'https://not.example2.com/otherpage.html',  b'https://example2.com/'),
+        ('http://example2.com/page.html',   'http://not.example2.com/otherpage.html',   b'http://example2.com/'),
+        # exact match required
+        ('http://example2.com/page.html',   'http://www.example2.com/otherpage.html',   b'http://example2.com/'),
+
+        # Different port: send origin as referrer
+        ('https://example3.com:444/page.html',  'https://example3.com/not-page.html',   b'https://example3.com:444/'),
+        ('http://example3.com:81/page.html',    'http://example3.com/not-page.html',    b'http://example3.com:81/'),
+
+        # downgrade
+        ('https://example4.com/page.html',  'http://example4.com/not-page.html',    None),
+        ('https://example4.com/page.html',  'http://not.example4.com/',             None),
+
+        # non-TLS to non-TLS
+        ('ftp://example4.com/urls.zip',     'http://example4.com/not-page.html',    b'ftp://example4.com/'),
+
+        # upgrade
+        ('http://example4.com/page.html',  'https://example4.com/not-page.html',    b'http://example4.com/'),
+        ('http://example4.com/page.html',  'https://not.example4.com/',             b'http://example4.com/'),
+
+        # Different protocols: send origin as referrer
+        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+        ('ftps://example4.com/urls.zip',    'https://example4.com/not-page.html',   b'ftps://example4.com/'),
+
+        # test for user/password stripping
+        ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html',  b'https://example5.com/page.html'),
+
+        # TLS to non-TLS downgrade: send nothing
+        ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html',   None),
+    ]
+
+
+class MixinUnsafeUrl(object):
+    scenarii = [
+        # TLS to TLS: send referrer
+        ('https://example.com/sekrit.html',     'http://not.example.com/',      b'https://example.com/sekrit.html'),
+        ('https://example1.com/page.html',      'https://not.example1.com/',    b'https://example1.com/page.html'),
+        ('https://example1.com/page.html',      'https://scrapy.org/',          b'https://example1.com/page.html'),
+        ('https://example1.com:443/page.html',  'https://scrapy.org/',          b'https://example1.com/page.html'),
+        ('https://example1.com:444/page.html',  'https://scrapy.org/',          b'https://example1.com:444/page.html'),
+        ('ftps://example1.com/urls.zip',        'https://scrapy.org/',          b'ftps://example1.com/urls.zip'),
+
+        # TLS to non-TLS: send referrer (yes, it's unsafe)
+        ('https://example2.com/page.html',  'http://not.example2.com/', b'https://example2.com/page.html'),
+        ('https://example2.com/page.html',  'http://scrapy.org/',       b'https://example2.com/page.html'),
+        ('ftps://example2.com/urls.zip',    'http://scrapy.org/',       b'ftps://example2.com/urls.zip'),
+
+        # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
+        ('http://example3.com/page.html',       'https://not.example3.com/',    b'http://example3.com/page.html'),
+        ('http://example3.com/page.html',       'https://scrapy.org/',          b'http://example3.com/page.html'),
+        ('http://example3.com:8080/page.html',  'https://scrapy.org/',          b'http://example3.com:8080/page.html'),
+        ('http://example3.com:80/page.html',    'http://not.example3.com/',     b'http://example3.com/page.html'),
+        ('http://example3.com/page.html',       'http://scrapy.org/',           b'http://example3.com/page.html'),
+        ('http://example3.com:443/page.html',   'http://scrapy.org/',           b'http://example3.com:443/page.html'),
+        ('ftp://example3.com/urls.zip',         'http://scrapy.org/',           b'ftp://example3.com/urls.zip'),
+        ('ftp://example3.com/urls.zip',         'https://scrapy.org/',          b'ftp://example3.com/urls.zip'),
+
+        # test for user/password stripping
+        ('http://user:password@example4.com/page.html',     'https://not.example4.com/',    b'http://example4.com/page.html'),
+        ('https://user:password@example4.com/page.html',    'http://scrapy.org/',           b'https://example4.com/page.html'),
+    ]
+
+
+class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):
+    pass
+
+
+# --- Tests using settings to set policy using class path
+class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}
+
+
+class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+
+
+class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+
+
+class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
+
+
+class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}
+
+
+class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+
+
+class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}
+
+
+class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
+
+
+class CustomPythonOrgPolicy(ReferrerPolicy):
+    """
+    A dummy policy that returns referrer as http(s)://python.org
+    depending on the scheme of the target URL.
+    """
+    def referrer(self, response, request):
+        scheme = urlparse(request).scheme
+        if scheme == 'https':
+            return b'https://python.org/'
+        elif scheme == 'http':
+            return b'http://python.org/'
+
+
+class TestSettingsCustomPolicy(TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
+    scenarii = [
+        ('https://example.com/',    'https://scrapy.org/',  b'https://python.org/'),
+        ('http://example.com/',     'http://scrapy.org/',   b'http://python.org/'),
+        ('http://example.com/',     'https://scrapy.org/',  b'https://python.org/'),
+        ('https://example.com/',    'http://scrapy.org/',   b'http://python.org/'),
+        ('file:///home/path/to/somefile.html',  'https://scrapy.org/', b'https://python.org/'),
+        ('file:///home/path/to/somefile.html',  'http://scrapy.org/',  b'http://python.org/'),
+
+    ]
+
+# --- Tests using Request meta dict to set policy
+class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
+
+
+class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_NO_REFERRER}
+
+
+class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}
+
+
+class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_SAME_ORIGIN}
+
+
+class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_ORIGIN}
+
+
+class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN}
+
+
+class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
+
+
+class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
+
+
+class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
+    req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+
+
+class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+    req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+
+
+class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+    req_meta = {'referrer_policy': POLICY_NO_REFERRER}
+
+
+class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+    req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+
+
+class TestRequestMetaSettingFallback(TestCase):
+
+    params = [
+        (
+            # When an unknown policy is referenced in Request.meta
+            # (here, a typo error),
+            # the policy defined in settings takes precedence
+            {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'},
+            {},
+            {'referrer_policy': 'ssscrapy-default'},
+            OriginWhenCrossOriginPolicy,
+            True
+        ),
+        (
+            # same as above but with string value for settings policy
+            {'REFERRER_POLICY': 'origin-when-cross-origin'},
+            {},
+            {'referrer_policy': 'ssscrapy-default'},
+            OriginWhenCrossOriginPolicy,
+            True
+        ),
+        (
+            # request meta references a wrong policy but it is set,
+            # so the Referrer-Policy header in response is not used,
+            # and the settings' policy is applied
+            {'REFERRER_POLICY': 'origin-when-cross-origin'},
+            {'Referrer-Policy': 'unsafe-url'},
+            {'referrer_policy': 'ssscrapy-default'},
+            OriginWhenCrossOriginPolicy,
+            True
+        ),
+        (
+            # here, request meta does not set the policy
+            # so response headers take precedence
+            {'REFERRER_POLICY': 'origin-when-cross-origin'},
+            {'Referrer-Policy': 'unsafe-url'},
+            {},
+            UnsafeUrlPolicy,
+            False
+        ),
+        (
+            # here, request meta does not set the policy,
+            # but response headers also use an unknown policy,
+            # so the settings' policy is used
+            {'REFERRER_POLICY': 'origin-when-cross-origin'},
+            {'Referrer-Policy': 'unknown'},
+            {},
+            OriginWhenCrossOriginPolicy,
+            True
+        )
+    ]
+
+    def test(self):
+
+        origin = 'http://www.scrapy.org'
+        target = 'http://www.example.com'
+
+        for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]:
+            spider = Spider('foo')
+            mw = RefererMiddleware(Settings(settings))
+
+            response = Response(origin, headers=response_headers)
+            request = Request(target, meta=request_meta)
+
+            with warnings.catch_warnings(record=True) as w:
+                policy = mw.policy(response, request)
+                self.assertIsInstance(policy, policy_class)
+
+                if check_warning:
+                    self.assertEqual(len(w), 1)
+                    self.assertEqual(w[0].category, RuntimeWarning, w[0].message)
+
+
+class TestSettingsPolicyByName(TestCase):
+
+    def test_valid_name(self):
+        for s, p in [
+                (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
+                (POLICY_NO_REFERRER, NoReferrerPolicy),
+                (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
+                (POLICY_SAME_ORIGIN, SameOriginPolicy),
+                (POLICY_ORIGIN, OriginPolicy),
+                (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
+                (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+                (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
+                (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
+            ]:
+            settings = Settings({'REFERRER_POLICY': s})
+            mw = RefererMiddleware(settings)
+            self.assertEquals(mw.default_policy, p)
+
+    def test_valid_name_casevariants(self):
+        for s, p in [
+                (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
+                (POLICY_NO_REFERRER, NoReferrerPolicy),
+                (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
+                (POLICY_SAME_ORIGIN, SameOriginPolicy),
+                (POLICY_ORIGIN, OriginPolicy),
+                (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
+                (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+                (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
+                (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
+            ]:
+            settings = Settings({'REFERRER_POLICY': s.upper()})
+            mw = RefererMiddleware(settings)
+            self.assertEquals(mw.default_policy, p)
+
+    def test_invalid_name(self):
+        settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
+        with self.assertRaises(RuntimeError):
+            mw = RefererMiddleware(settings)
+
+
+class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+    resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
+
+class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+    resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
+
+class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+    resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
+
+
+class TestReferrerOnRedirect(TestRefererMiddleware):
+
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
+    scenarii = [
+        (   'http://scrapytest.org/1',      # parent
+            'http://scrapytest.org/2',      # target
+            (
+                # redirections: code, URL
+                (301, 'http://scrapytest.org/3'),
+                (301, 'http://scrapytest.org/4'),
+            ),
+            b'http://scrapytest.org/1', # expected initial referer
+            b'http://scrapytest.org/1', # expected referer for the redirection request
+        ),
+        (   'https://scrapytest.org/1',
+            'https://scrapytest.org/2',
+            (
+                # redirecting to non-secure URL
+                (301, 'http://scrapytest.org/3'),
+            ),
+            b'https://scrapytest.org/1',
+            b'https://scrapytest.org/1',
+        ),
+        (   'https://scrapytest.org/1',
+            'https://scrapytest.com/2',
+            (
+                # redirecting to non-secure URL: different origin
+                (301, 'http://scrapytest.com/3'),
+            ),
+            b'https://scrapytest.org/1',
+            b'https://scrapytest.org/1',
+        ),
+    ]
+
+    def setUp(self):
+        self.spider = Spider('foo')
+        settings = Settings(self.settings)
+        self.referrermw = RefererMiddleware(settings)
+        self.redirectmw = RedirectMiddleware(settings)
+
+    def test(self):
+
+        for parent, target, redirections, init_referrer, final_referrer in self.scenarii:
+            response = self.get_response(parent)
+            request = self.get_request(target)
+
+            out = list(self.referrermw.process_spider_output(response, [request], self.spider))
+            self.assertEquals(out[0].headers.get('Referer'), init_referrer)
+
+            for status, url in redirections:
+                response = Response(request.url, headers={'Location': url}, status=status)
+                request = self.redirectmw.process_response(request, response, self.spider)
+                self.referrermw.request_scheduled(request, self.spider)
+
+            assert isinstance(request, Request)
+            self.assertEquals(request.headers.get('Referer'), final_referrer)
+
+
+class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
+    """
+    No Referrer policy never sets the "Referer" header.
+    HTTP redirections should not change that.
+    """
+    settings = {'REFERRER_POLICY': 'no-referrer'}
+    scenarii = [
+        (   'http://scrapytest.org/1',      # parent
+            'http://scrapytest.org/2',      # target
+            (
+                # redirections: code, URL
+                (301, 'http://scrapytest.org/3'),
+                (301, 'http://scrapytest.org/4'),
+            ),
+            None, # expected initial "Referer"
+            None, # expected "Referer" for the redirection request
+        ),
+        (   'https://scrapytest.org/1',
+            'https://scrapytest.org/2',
+            (
+                (301, 'http://scrapytest.org/3'),
+            ),
+            None,
+            None,
+        ),
+        (   'https://scrapytest.org/1',
+            'https://example.com/2',    # different origin
+            (
+                (301, 'http://scrapytest.com/3'),
+            ),
+            None,
+            None,
+        ),
+    ]
+
+
+class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
+    """
+    Same Origin policy sends the full URL as "Referer" if the target origin
+    is the same as the parent response (same protocol, same domain, same port).
+
+    HTTP redirections to a different domain or a lower secure level
+    should have the "Referer" removed.
+    """
+    settings = {'REFERRER_POLICY': 'same-origin'}
+    scenarii = [
+        (   'http://scrapytest.org/101',      # origin
+            'http://scrapytest.org/102',      # target
+            (
+                # redirections: code, URL
+                (301, 'http://scrapytest.org/103'),
+                (301, 'http://scrapytest.org/104'),
+            ),
+            b'http://scrapytest.org/101', # expected initial "Referer"
+            b'http://scrapytest.org/101', # expected referer for the redirection request
+        ),
+        (   'https://scrapytest.org/201',
+            'https://scrapytest.org/202',
+            (
+                # redirecting from secure to non-secure URL == different origin
+                (301, 'http://scrapytest.org/203'),
+            ),
+            b'https://scrapytest.org/201',
+            None,
+        ),
+        (   'https://scrapytest.org/301',
+            'https://scrapytest.org/302',
+            (
+                # different domain == different origin
+                (301, 'http://example.com/303'),
+            ),
+            b'https://scrapytest.org/301',
+            None,
+        ),
+    ]
+
+
+class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
+    """
+    Strict Origin policy will always send the "origin" as referrer
+    (think of it as the parent URL without the path part),
+    unless the security level is lower and no "Referer" is sent.
+
+    Redirections from secure to non-secure URLs should have the
+    "Referrer" header removed if necessary.
+    """
+    settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN}
+    scenarii = [
+        (   'http://scrapytest.org/101',
+            'http://scrapytest.org/102',
+            (
+                (301, 'http://scrapytest.org/103'),
+                (301, 'http://scrapytest.org/104'),
+            ),
+            b'http://scrapytest.org/',  # send origin
+            b'http://scrapytest.org/',  # redirects to same origin: send origin
+        ),
+        (   'https://scrapytest.org/201',
+            'https://scrapytest.org/202',
+            (
+                # redirecting to non-secure URL: no referrer
+                (301, 'http://scrapytest.org/203'),
+            ),
+            b'https://scrapytest.org/',
+            None,
+        ),
+        (   'https://scrapytest.org/301',
+            'https://scrapytest.org/302',
+            (
+                # redirecting to non-secure URL (different domain): no referrer
+                (301, 'http://example.com/303'),
+            ),
+            b'https://scrapytest.org/',
+            None,
+        ),
+        (   'http://scrapy.org/401',
+            'http://example.com/402',
+            (
+                (301, 'http://scrapytest.org/403'),
+            ),
+            b'http://scrapy.org/',
+            b'http://scrapy.org/',
+        ),
+        (   'https://scrapy.org/501',
+            'https://example.com/502',
+            (
+                # HTTPS all along, so origin referrer is kept as-is
+                (301, 'https://google.com/503'),
+                (301, 'https://facebook.com/504'),
+            ),
+            b'https://scrapy.org/',
+            b'https://scrapy.org/',
+        ),
+        (   'https://scrapytest.org/601',
+            'http://scrapytest.org/602',                # TLS to non-TLS: no referrer
+            (
+                (301, 'https://scrapytest.org/603'),    # TLS URL again: (still) no referrer
+            ),
+            None,
+            None,
+        ),
+    ]
+
+
+class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
+    """
+    Origin When Cross-Origin policy sends the full URL as "Referer",
+    unless the target's origin is different (different domain, different protocol)
+    in which case only the origin is sent.
+
+    Redirections to a different origin should strip the "Referer"
+    to the parent origin.
+    """
+    settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
+    scenarii = [
+        (   'http://scrapytest.org/101',      # origin
+            'http://scrapytest.org/102',      # target + redirection
+            (
+                # redirections: code, URL
+                (301, 'http://scrapytest.org/103'),
+                (301, 'http://scrapytest.org/104'),
+            ),
+            b'http://scrapytest.org/101', # expected initial referer
+            b'http://scrapytest.org/101', # expected referer for the redirection request
+        ),
+        (   'https://scrapytest.org/201',
+            'https://scrapytest.org/202',
+            (
+                # redirecting to non-secure URL: send origin
+                (301, 'http://scrapytest.org/203'),
+            ),
+            b'https://scrapytest.org/201',
+            b'https://scrapytest.org/',
+        ),
+        (   'https://scrapytest.org/301',
+            'https://scrapytest.org/302',
+            (
+                # redirecting to non-secure URL (different domain): send origin
+                (301, 'http://example.com/303'),
+            ),
+            b'https://scrapytest.org/301',
+            b'https://scrapytest.org/',
+        ),
+        (   'http://scrapy.org/401',
+            'http://example.com/402',
+            (
+                (301, 'http://scrapytest.org/403'),
+            ),
+            b'http://scrapy.org/',
+            b'http://scrapy.org/',
+        ),
+        (   'https://scrapy.org/501',
+            'https://example.com/502',
+            (
+                # all different domains: send origin
+                (301, 'https://google.com/503'),
+                (301, 'https://facebook.com/504'),
+            ),
+            b'https://scrapy.org/',
+            b'https://scrapy.org/',
+        ),
+        (   'https://scrapytest.org/301',
+            'http://scrapytest.org/302',                # TLS to non-TLS: send origin
+            (
+                (301, 'https://scrapytest.org/303'),    # TLS URL again: send origin (also)
+            ),
+            b'https://scrapytest.org/',
+            b'https://scrapytest.org/',
+        ),
+    ]
 
-    def test_process_spider_output(self):
-        res = Response('http://scrapytest.org')
-        reqs = [Request('http://scrapytest.org/')]
 
-        out = list(self.mw.process_spider_output(res, reqs, self.spider))
-        self.assertEquals(out[0].headers.get('Referer'),
-                          b'http://scrapytest.org')
+class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
+    """
+    Strict Origin When Cross-Origin policy sends the full URL as "Referer",
+    unless the target's origin is different (different domain, different protocol)
+    in which case only the origin is sent...
+    Unless there's also a downgrade in security and then the "Referer" header
+    is not sent.
 
+    Redirections to a different origin should strip the "Referer" to the parent origin,
+    and from https:// to http:// will remove the "Referer" header.
+    """
+    settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
+    scenarii = [
+        (   'http://scrapytest.org/101',      # origin
+            'http://scrapytest.org/102',      # target + redirection
+            (
+                # redirections: code, URL
+                (301, 'http://scrapytest.org/103'),
+                (301, 'http://scrapytest.org/104'),
+            ),
+            b'http://scrapytest.org/101', # expected initial referer
+            b'http://scrapytest.org/101', # expected referer for the redirection request
+        ),
+        (   'https://scrapytest.org/201',
+            'https://scrapytest.org/202',
+            (
+                # redirecting to non-secure URL: do not send the "Referer" header
+                (301, 'http://scrapytest.org/203'),
+            ),
+            b'https://scrapytest.org/201',
+            None,
+        ),
+        (   'https://scrapytest.org/301',
+            'https://scrapytest.org/302',
+            (
+                # redirecting to non-secure URL (different domain): send origin
+                (301, 'http://example.com/303'),
+            ),
+            b'https://scrapytest.org/301',
+            None,
+        ),
+        (   'http://scrapy.org/401',
+            'http://example.com/402',
+            (
+                (301, 'http://scrapytest.org/403'),
+            ),
+            b'http://scrapy.org/',
+            b'http://scrapy.org/',
+        ),
+        (   'https://scrapy.org/501',
+            'https://example.com/502',
+            (
+                # all different domains: send origin
+                (301, 'https://google.com/503'),
+                (301, 'https://facebook.com/504'),
+            ),
+            b'https://scrapy.org/',
+            b'https://scrapy.org/',
+        ),
+        (   'https://scrapytest.org/601',
+            'http://scrapytest.org/602',                # TLS to non-TLS: do not send "Referer"
+            (
+                (301, 'https://scrapytest.org/603'),    # TLS URL again: (still) send nothing
+            ),
+            None,
+            None,
+        ),
+    ]
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -6,7 +6,8 @@
 
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
-                              add_http_if_no_scheme, guess_scheme, parse_url)
+                              add_http_if_no_scheme, guess_scheme,
+                              parse_url, strip_url)
 
 __doctests__ = ['scrapy.utils.url']
 
@@ -241,5 +242,171 @@ def do_expected(self):
     setattr (GuessSchemeTest, t_method.__name__, t_method)
 
 
+class StripUrl(unittest.TestCase):
+
+    def test_noop(self):
+        self.assertEqual(strip_url(
+            'http://www.example.com/index.html'),
+            'http://www.example.com/index.html')
+
+    def test_noop_query_string(self):
+        self.assertEqual(strip_url(
+            'http://www.example.com/index.html?somekey=somevalue'),
+            'http://www.example.com/index.html?somekey=somevalue')
+
+    def test_fragments(self):
+        self.assertEqual(strip_url(
+            'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False),
+            'http://www.example.com/index.html?somekey=somevalue#section')
+
+    def test_path(self):
+        for input_url, origin, output_url in [
+            ('http://www.example.com/',
+             False,
+             'http://www.example.com/'),
+
+            ('http://www.example.com',
+             False,
+             'http://www.example.com'),
+
+            ('http://www.example.com',
+             True,
+             'http://www.example.com/'),
+            ]:
+            self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
+
+    def test_credentials(self):
+        for i, o in [
+            ('http://username@www.example.com/index.html?somekey=somevalue#section',
+             'http://www.example.com/index.html?somekey=somevalue'),
+
+            ('https://username:@www.example.com/index.html?somekey=somevalue#section',
+             'https://www.example.com/index.html?somekey=somevalue'),
+
+            ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
+             'ftp://www.example.com/index.html?somekey=somevalue'),
+            ]:
+            self.assertEqual(strip_url(i, strip_credentials=True), o)
+
+    def test_credentials_encoded_delims(self):
+        for i, o in [
+            # user: "username@"
+            # password: none
+            ('http://username%40@www.example.com/index.html?somekey=somevalue#section',
+             'http://www.example.com/index.html?somekey=somevalue'),
+
+            # user: "username:pass"
+            # password: ""
+            ('https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section',
+             'https://www.example.com/index.html?somekey=somevalue'),
+
+            # user: "me"
+            # password: "user@domain.com"
+            ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section',
+             'ftp://www.example.com/index.html?somekey=somevalue'),
+            ]:
+            self.assertEqual(strip_url(i, strip_credentials=True), o)
+
+    def test_default_ports_creds_off(self):
+        for i, o in [
+            ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section',
+             'http://www.example.com/index.html?somekey=somevalue'),
+
+            ('http://username:password@www.example.com:8080/index.html#section',
+             'http://www.example.com:8080/index.html'),
+
+            ('http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section',
+             'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'),
+
+            ('https://username:password@www.example.com:443/index.html',
+             'https://www.example.com/index.html'),
+
+            ('https://username:password@www.example.com:442/index.html',
+             'https://www.example.com:442/index.html'),
+
+            ('https://username:password@www.example.com:80/index.html',
+             'https://www.example.com:80/index.html'),
+
+            ('ftp://username:password@www.example.com:21/file.txt',
+             'ftp://www.example.com/file.txt'),
+
+            ('ftp://username:password@www.example.com:221/file.txt',
+             'ftp://www.example.com:221/file.txt'),
+            ]:
+            self.assertEqual(strip_url(i), o)
+
+    def test_default_ports(self):
+        for i, o in [
+            ('http://username:password@www.example.com:80/index.html',
+             'http://username:password@www.example.com/index.html'),
+
+            ('http://username:password@www.example.com:8080/index.html',
+             'http://username:password@www.example.com:8080/index.html'),
+
+            ('http://username:password@www.example.com:443/index.html',
+             'http://username:password@www.example.com:443/index.html'),
+
+            ('https://username:password@www.example.com:443/index.html',
+             'https://username:password@www.example.com/index.html'),
+
+            ('https://username:password@www.example.com:442/index.html',
+             'https://username:password@www.example.com:442/index.html'),
+
+            ('https://username:password@www.example.com:80/index.html',
+             'https://username:password@www.example.com:80/index.html'),
+
+            ('ftp://username:password@www.example.com:21/file.txt',
+             'ftp://username:password@www.example.com/file.txt'),
+
+            ('ftp://username:password@www.example.com:221/file.txt',
+             'ftp://username:password@www.example.com:221/file.txt'),
+            ]:
+            self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
+
+    def test_default_ports_keep(self):
+        for i, o in [
+            ('http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section',
+             'http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov'),
+
+            ('http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section',
+             'http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov'),
+
+            ('http://username:password@www.example.com:443/index.html',
+             'http://username:password@www.example.com:443/index.html'),
+
+            ('https://username:password@www.example.com:443/index.html',
+             'https://username:password@www.example.com:443/index.html'),
+
+            ('https://username:password@www.example.com:442/index.html',
+             'https://username:password@www.example.com:442/index.html'),
+
+            ('https://username:password@www.example.com:80/index.html',
+             'https://username:password@www.example.com:80/index.html'),
+
+            ('ftp://username:password@www.example.com:21/file.txt',
+             'ftp://username:password@www.example.com:21/file.txt'),
+
+            ('ftp://username:password@www.example.com:221/file.txt',
+             'ftp://username:password@www.example.com:221/file.txt'),
+            ]:
+            self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
+
+    def test_origin_only(self):
+        for i, o in [
+            ('http://username:password@www.example.com/index.html',
+             'http://www.example.com/'),
+
+            ('http://username:password@www.example.com:80/foo/bar?query=value#somefrag',
+             'http://www.example.com/'),
+
+            ('http://username:password@www.example.com:8008/foo/bar?query=value#somefrag',
+             'http://www.example.com:8008/'),
+
+            ('https://username:password@www.example.com:443/index.html',
+             'https://www.example.com/'),
+            ]:
+            self.assertEqual(strip_url(i, origin_only=True), o)
+
+
 if __name__ == "__main__":
     unittest.main()