Repository: scrapy/scrapy

Issue:
`scrapy.FormRequest.from_response` observed to eliminate duplicate keys in `formdata`
This looks good:
```
In [2]: scrapy.FormRequest('http://example.com', method='GET', formdata=(('foo', 'bar'), ('foo', 'baz')))
Out[2]: <GET http://example.com?foo=bar&foo=baz>
```

While here is the issue:
```
In [3]: response = scrapy.http.TextResponse(url='http://example.com', body='<form></form>', encoding='utf8')

In [4]: scrapy.FormRequest.from_response(response, method='GET', formdata=(('foo', 'bar'), ('foo', 'baz')))
Out[4]: <GET http://example.com?foo=baz>
```

(Tested with `Scrapy 1.5.0` and `Python 3.6.5`)

Comments:

Patch:
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index a54b4daf0..eb790a67e 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -4,7 +4,7 @@ import logging
 from os.path import join, exists
 
 from scrapy.utils.reqser import request_to_dict, request_from_dict
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.job import job_dir
 
 logger = logging.getLogger(__name__)
@@ -26,7 +26,7 @@ class Scheduler(object):
     def from_crawler(cls, crawler):
         settings = crawler.settings
         dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
-        dupefilter = dupefilter_cls.from_settings(settings)
+        dupefilter = create_instance(dupefilter_cls, settings, crawler)
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 95b38e990..c2413b431 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -114,10 +114,12 @@ def _get_form(response, formname, formid, formnumber, formxpath):
 
 def _get_inputs(form, formdata, dont_click, clickdata, response):
     try:
-        formdata = dict(formdata or ())
+        formdata_keys = dict(formdata or ()).keys()
     except (ValueError, TypeError):
         raise ValueError('formdata should be a dict or iterable of tuples')
 
+    if not formdata:
+        formdata = ()
     inputs = form.xpath('descendant::textarea'
                         '|descendant::select'
                         '|descendant::input[not(@type) or @type['
@@ -128,14 +130,17 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
                             "re": "http://exslt.org/regular-expressions"})
     values = [(k, u'' if v is None else v)
               for k, v in (_value(e) for e in inputs)
-              if k and k not in formdata]
+              if k and k not in formdata_keys]
 
     if not dont_click:
         clickable = _get_clickable(clickdata, form)
         if clickable and clickable[0] not in formdata and not clickable[0] is None:
             values.append(clickable)
 
-    values.extend((k, v) for k, v in formdata.items() if v is not None)
+    if isinstance(formdata, dict):
+        formdata = formdata.items()
+
+    values.extend((k, v) for k, v in formdata if v is not None)
     return values
 
 


Test patch:
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 2d1a4bfff..db69597a2 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -5,11 +5,60 @@ import shutil
 
 from scrapy.dupefilters import RFPDupeFilter
 from scrapy.http import Request
+from scrapy.core.scheduler import Scheduler
 from scrapy.utils.python import to_bytes
+from scrapy.utils.job import job_dir
+from scrapy.utils.test import get_crawler
+
+
+class FromCrawlerRFPDupeFilter(RFPDupeFilter):
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
+        df = cls(job_dir(crawler.settings), debug)
+        df.method = 'from_crawler'
+        return df
+
+
+class FromSettingsRFPDupeFilter(RFPDupeFilter):
+
+    @classmethod
+    def from_settings(cls, settings):
+        debug = settings.getbool('DUPEFILTER_DEBUG')
+        df = cls(job_dir(settings), debug)
+        df.method = 'from_settings'
+        return df
+
+
+class DirectDupeFilter(object):
+    method = 'n/a'
 
 
 class RFPDupeFilterTest(unittest.TestCase):
 
+    def test_df_from_crawler_scheduler(self):
+        settings = {'DUPEFILTER_DEBUG': True,
+                    'DUPEFILTER_CLASS': __name__  + '.FromCrawlerRFPDupeFilter'}
+        crawler = get_crawler(settings_dict=settings)
+        scheduler = Scheduler.from_crawler(crawler)
+        self.assertTrue(scheduler.df.debug)
+        self.assertEqual(scheduler.df.method, 'from_crawler')
+
+    def test_df_from_settings_scheduler(self):
+        settings = {'DUPEFILTER_DEBUG': True,
+                    'DUPEFILTER_CLASS': __name__  + '.FromSettingsRFPDupeFilter'}
+        crawler = get_crawler(settings_dict=settings)
+        scheduler = Scheduler.from_crawler(crawler)
+        self.assertTrue(scheduler.df.debug)
+        self.assertEqual(scheduler.df.method, 'from_settings')
+
+    def test_df_direct_scheduler(self):
+        settings = {'DUPEFILTER_CLASS': __name__  + '.DirectDupeFilter'}
+        crawler = get_crawler(settings_dict=settings)
+        scheduler = Scheduler.from_crawler(crawler)
+        self.assertEqual(scheduler.df.method, 'n/a')
+
     def test_filter(self):
         dupefilter = RFPDupeFilter()
         dupefilter.open()
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index fc89229c6..58326a384 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -406,6 +406,29 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[u'test2'], [u'xxx µ'])
         self.assertEqual(fs[u'six'], [u'seven'])
 
+    def test_from_response_duplicate_form_key(self):
+        response = _buildresponse(
+                '<form></form>',
+                url='http://www.example.com')
+        req = self.request_class.from_response(response,
+                method='GET',
+                formdata=(('foo', 'bar'), ('foo', 'baz')))
+        self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
+        self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
+    
+    def test_from_response_override_duplicate_form_key(self):
+        response = _buildresponse(
+            """<form action="get.php" method="POST">
+            <input type="hidden" name="one" value="1">
+            <input type="hidden" name="two" value="3">
+            </form>""")
+        req = self.request_class.from_response(
+            response,
+            formdata=(('two', '2'), ('two', '4')))
+        fs = _qs(req)
+        self.assertEqual(fs[b'one'], [b'1'])
+        self.assertEqual(fs[b'two'], [b'2', b'4'])
+
     def test_from_response_extra_headers(self):
         response = _buildresponse(
             """<form action="post.php" method="POST">