Repository: scrapy/scrapy

Issue:
`scrapy.FormRequest.from_response` observed to eliminate duplicate keys in `formdata`
This looks good:
```
In [2]: scrapy.FormRequest('http://example.com', method='GET', formdata=(('foo', 'bar'), ('foo', 'baz')))
Out[2]: <GET http://example.com?foo=bar&foo=baz>
```

While here is the issue:
```
In [3]: response = scrapy.http.TextResponse(url='http://example.com', body='<form></form>', encoding='utf8')

In [4]: scrapy.FormRequest.from_response(response, method='GET', formdata=(('foo', 'bar'), ('foo', 'baz')))
Out[4]: <GET http://example.com?foo=baz>
```

(Tested with `Scrapy 1.5.0` and `Python 3.6.5`)


Discussion:
Related code: https://github.com/scrapy/scrapy/blob/dd064413a46356940151ac3f9ccd8a45bca2cbd8/scrapy/http/request/form.py#L117
I'll take a look if no one's jumped on it yet.

Resolving patch:
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -114,10 +114,12 @@ def _get_form(response, formname, formid, formnumber, formxpath):
 
 def _get_inputs(form, formdata, dont_click, clickdata, response):
     try:
-        formdata = dict(formdata or ())
+        formdata_keys = dict(formdata or ()).keys()
     except (ValueError, TypeError):
         raise ValueError('formdata should be a dict or iterable of tuples')
 
+    if not formdata:
+        formdata = ()
     inputs = form.xpath('descendant::textarea'
                         '|descendant::select'
                         '|descendant::input[not(@type) or @type['
@@ -128,14 +130,17 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
                             "re": "http://exslt.org/regular-expressions"})
     values = [(k, u'' if v is None else v)
               for k, v in (_value(e) for e in inputs)
-              if k and k not in formdata]
+              if k and k not in formdata_keys]
 
     if not dont_click:
         clickable = _get_clickable(clickdata, form)
         if clickable and clickable[0] not in formdata and not clickable[0] is None:
             values.append(clickable)
 
-    values.extend((k, v) for k, v in formdata.items() if v is not None)
+    if isinstance(formdata, dict):
+        formdata = formdata.items()
+
+    values.extend((k, v) for k, v in formdata if v is not None)
     return values
 
 


Test patch:
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -401,6 +401,29 @@ def test_from_response_post_nonascii_unicode(self):
         self.assertEqual(fs[u'test2'], [u'xxx µ'])
         self.assertEqual(fs[u'six'], [u'seven'])
 
+    def test_from_response_duplicate_form_key(self):
+        response = _buildresponse(
+                '<form></form>',
+                url='http://www.example.com')
+        req = self.request_class.from_response(response,
+                method='GET',
+                formdata=(('foo', 'bar'), ('foo', 'baz')))
+        self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
+        self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
+    
+    def test_from_response_override_duplicate_form_key(self):
+        response = _buildresponse(
+            """<form action="get.php" method="POST">
+            <input type="hidden" name="one" value="1">
+            <input type="hidden" name="two" value="3">
+            </form>""")
+        req = self.request_class.from_response(
+            response,
+            formdata=(('two', '2'), ('two', '4')))
+        fs = _qs(req)
+        self.assertEqual(fs[b'one'], [b'1'])
+        self.assertEqual(fs[b'two'], [b'2', b'4'])
+
     def test_from_response_extra_headers(self):
         response = _buildresponse(
             """<form action="post.php" method="POST">