Repository: scrapy/w3lib

Issue:
url_query_cleaner appends ? to urls without a query string
```python
>>> url_query_cleaner('http://domain.tld/', ['bla'], remove=True)
'http://domain.tld/?'
```

This is the code which does it: `url = '?'.join([base, sep.join(querylist)]) if querylist else base` (`querylist==['']`).

Discussion:

Patch:
diff --git a/w3lib/url.py b/w3lib/url.py
index 4be74f7..fc9b343 100644
--- a/w3lib/url.py
+++ b/w3lib/url.py
@@ -182,6 +182,8 @@ def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='=', remove=False, u
     seen = set()
     querylist = []
     for ksv in query.split(sep):
+        if not ksv:
+            continue
         k, _, _ = ksv.partition(kvsep)
         if unique and k in seen:
             continue


Test patch:
diff --git a/tests/test_url.py b/tests/test_url.py
index 0df5bfd..9476b30 100644
--- a/tests/test_url.py
+++ b/tests/test_url.py
@@ -284,6 +284,10 @@ class UrlTests(unittest.TestCase):
                          'http://example.com/?version=1&pageurl=test&param2=value2')
 
     def test_url_query_cleaner(self):
+        self.assertEqual('product.html',
+                url_query_cleaner("product.html?"))
+        self.assertEqual('product.html',
+                url_query_cleaner("product.html?&"))
         self.assertEqual('product.html?id=200',
                 url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id']))
         self.assertEqual('product.html?id=200',
@@ -308,6 +312,10 @@ class UrlTests(unittest.TestCase):
                 url_query_cleaner("product.html?id=2&foo=bar&name=wired", ['id', 'foo'], remove=True))
         self.assertEqual('product.html?foo=bar&name=wired',
                 url_query_cleaner("product.html?id=2&foo=bar&name=wired", ['id', 'footo'], remove=True))
+        self.assertEqual('product.html',
+                url_query_cleaner("product.html", ['id'], remove=True))
+        self.assertEqual('product.html',
+                url_query_cleaner("product.html?&", ['id'], remove=True))
         self.assertEqual('product.html?foo=bar',
                 url_query_cleaner("product.html?foo=bar&name=wired", 'foo'))
         self.assertEqual('product.html?foobar=wired',