Repository: scrapy/scrapy

Issue:
Ability to not send specific headers in HTTP requests
Some web servers behave differently when they are receive or don't receive specific headers.

For example FeedBurner (http://feeds.feedburner.com/someblog) sends out XML RSS feeds **only is you do not set the "Referer" header.**

The idea would be to use the `headers` dict with some keys with a `None` value, and skip these headers when sending the HTTP request.

Currently, for the "Referer" example:
- `headers={"Referer": None}` sends "Referer: None"
- `headers={"Referer": ""}` sends "Referer: " (which works for the FeedBurner case, but is not satisfactory)
- disable `RefererMiddleware` but that feels a bit heavy

(for this FeedBurner thing, apparently adding `?format=xml` also does the trick)



Discussion:
it's sounds good to me

An alternative (more customary) approach for this specific issue would be to add a `dont_refer` flag to requests.meta, but this generic approach  to shut down headers sounds good to me too.

Both sound good to me (simple meta + alternative expert mode)

Can we implement this on a Middleware? Since this is not a required feature for scrapy to work.

- urllib2: None is cast to string. I didn't find an easy way to remove standard headers with `urllib2.Request` + `urllib2.urlopen` combo. Custom headers (that are set by user) can be removed using `del req.headers['header-name']`. 
- It seems that Twisted also casts None to string.
- requests package removes headers set to None if they are passed to `requests.get`, `requests.post`, etc. But it doesn't remove them if they are used in `requests.Request` constructor or when value is assigned to request header after instantiation. Custom headers can be removed using `del req.headers['header-name']` in this case, standard headers can't be removed.


Resolving patch:
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -17,7 +17,9 @@ def normkey(self, key):
 
     def normvalue(self, value):
         """Headers must not be unicode"""
-        if not hasattr(value, '__iter__'):
+        if value is None:
+            value = []
+        elif not hasattr(value, '__iter__'):
             value = [value]
         return [x.encode(self.encoding) if isinstance(x, unicode) else x \
             for x in value]


Test patch:
diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py
--- a/scrapy/tests/mockserver.py
+++ b/scrapy/tests/mockserver.py
@@ -1,5 +1,5 @@
 from __future__ import print_function
-import sys, time, random, urllib, os
+import sys, time, random, urllib, os, json
 from subprocess import Popen, PIPE
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
@@ -119,6 +119,16 @@ def _delayedRender(self, request):
         request.finish()
 
 
+class Echo(LeafResource):
+
+    def render_GET(self, request):
+        output = {
+            'headers': dict(request.requestHeaders.getAllRawHeaders()),
+            'body': request.content.read(),
+        }
+        return json.dumps(output)
+
+
 class Partial(LeafResource):
 
     def render_GET(self, request):
@@ -156,6 +166,7 @@ def __init__(self):
         self.putChild("partial", Partial())
         self.putChild("drop", Drop())
         self.putChild("raw", Raw())
+        self.putChild("echo", Echo())
 
     def getChild(self, name, request):
         return self
diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py
--- a/scrapy/tests/spiders.py
+++ b/scrapy/tests/spiders.py
@@ -132,3 +132,22 @@ def parse(self, response):
         self.seedsseen.append(response.meta.get('seed'))
         for req in super(BrokenStartRequestsSpider, self).parse(response):
             yield req
+
+
+class SingleRequestSpider(MetaSpider):
+
+    seed = None
+
+    def start_requests(self):
+        if isinstance(self.seed, Request):
+            yield self.seed.replace(callback=self.parse, errback=self.on_error)
+        else:
+            yield Request(self.seed, callback=self.parse, errback=self.on_error)
+
+    def parse(self, response):
+        self.meta.setdefault('responses', []).append(response)
+        if 'next' in response.meta:
+            return response.meta['next']
+
+    def on_error(self, failure):
+        self.meta['failure'] = failure
diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py
--- a/scrapy/tests/test_crawl.py
+++ b/scrapy/tests/test_crawl.py
@@ -1,9 +1,11 @@
+import json
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from scrapy.utils.test import get_crawler, get_testlog
 from scrapy.tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
-    BrokenStartRequestsSpider
+    BrokenStartRequestsSpider, SingleRequestSpider
 from scrapy.tests.mockserver import MockServer
+from scrapy.http import Request
 
 
 def docrawl(spider, settings=None):
@@ -158,3 +160,31 @@ def _assert_retried(self):
         log = get_testlog()
         self.assertEqual(log.count("Retrying"), 2)
         self.assertEqual(log.count("Gave up retrying"), 1)
+
+    @defer.inlineCallbacks
+    def test_referer_header(self):
+        """Referer header is set by RefererMiddleware unless it is already set"""
+        req0 = Request('http://localhost:8998/echo?headers=1&body=0', dont_filter=1)
+        req1 = req0.replace()
+        req2 = req0.replace(headers={'Referer': None})
+        req3 = req0.replace(headers={'Referer': 'http://example.com'})
+        req0.meta['next'] = req1
+        req1.meta['next'] = req2
+        req2.meta['next'] = req3
+        spider = SingleRequestSpider(seed=req0)
+        yield docrawl(spider)
+        # basic asserts in case of weird communication errors
+        self.assertIn('responses', spider.meta)
+        self.assertNotIn('failures', spider.meta)
+        # start requests doesn't set Referer header
+        echo0 = json.loads(spider.meta['responses'][2].body)
+        self.assertNotIn('Referer', echo0['headers'])
+        # following request sets Referer to start request url
+        echo1 = json.loads(spider.meta['responses'][1].body)
+        self.assertEqual(echo1['headers'].get('Referer'), [req0.url])
+        # next request avoids Referer header
+        echo2 = json.loads(spider.meta['responses'][2].body)
+        self.assertNotIn('Referer', echo2['headers'])
+        # last request explicitly sets a Referer header
+        echo3 = json.loads(spider.meta['responses'][3].body)
+        self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com'])
diff --git a/scrapy/tests/test_http_headers.py b/scrapy/tests/test_http_headers.py
--- a/scrapy/tests/test_http_headers.py
+++ b/scrapy/tests/test_http_headers.py
@@ -119,3 +119,11 @@ def test_setlistdefault(self):
         h1.setlistdefault('header2', ['value2', 'value3'])
         self.assertEqual(h1.getlist('header1'), ['value1'])
         self.assertEqual(h1.getlist('header2'), ['value2', 'value3'])
+
+    def test_none_value(self):
+        h1 = Headers()
+        h1['foo'] = 'bar'
+        h1['foo'] = None
+        h1.setdefault('foo', 'bar')
+        self.assertEqual(h1.get('foo'), None)
+        self.assertEqual(h1.getlist('foo'), [])