Repository: scrapy/scrapy
Pull request #5450 (2022-03-16 14:29:15)

fixed detection of extensions like ".tar.gz" in URL
#4066 added more extensions to ignore by default such as `.rar` or `.zip`, which is great. 

Unfortunately, the addition of `.tar.gz` as an extension to exclude is not working because of the implementation of the function `url_has_any_extension` which can be found in `scrapy/utils/url.py`.

The actual function looks like this:

```python3
def url_has_any_extension(url, extensions):
    return posixpath.splitext(parse_url(url).path)[1].lower() in extensions
```

Here is an example that does not work:
```python3
import posixpath
from urllib.parse import urlparse
from scrapy.utils.url import parse_url

parsed_url = urlparse("https://dl.google.com/go/go1.7rc3.freebsd-amd64.tar.gz")
# ParseResult(scheme='https', netloc='dl.google.com', path='/go/go1.7rc3.freebsd-amd64.tar.gz', params='', query='', fragment='')

url_has_any_extension(parsed_url, [".tar.gz"])
# False

# Breaking down "url_has_any_extension" function
posixpath.splitext(parse_url(parsed_url).path)[1].lower() in [".tar.gz"]
# False

posixpath.splitext(parse_url(parsed_url).path)[1].lower()
# '.gz'

posixpath.splitext(parse_url(parsed_url).path)
# ('/go/go1.7rc3.freebsd-amd64.tar', '.gz')
```

We can see that the function splits the path from the first dot encountered, starting from the right, which leads to a `.tar.gz` extension not detected.

I fixed the function by making use of `any` and the standard Python method `endswith`.



Patch:
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index a6a2a9e8b..4d5e9ae82 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -5,7 +5,6 @@ library.
 Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
-import posixpath
 import re
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
@@ -31,8 +30,9 @@ def url_is_from_spider(url, spider):
 
 
 def url_has_any_extension(url, extensions):
-    return posixpath.splitext(parse_url(url).path)[1].lower() in extensions
-
+    """Return True if the url ends with one of the extensions provided"""
+    lowercase_path = parse_url(url).path.lower()
+    return any(lowercase_path.endswith(ext) for ext in extensions)
 
 def parse_url(url, encoding=None):
     """Return urlparsed url from the given argument (which could be an already


Test patch:
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 144c7bd76..58e2be622 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,6 +1,8 @@
 import unittest
 
+from scrapy.linkextractors import IGNORED_EXTENSIONS
 from scrapy.spiders import Spider
+from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.url import (
     add_http_if_no_scheme,
     guess_scheme,
@@ -8,9 +10,9 @@ from scrapy.utils.url import (
     strip_url,
     url_is_from_any_domain,
     url_is_from_spider,
+    url_has_any_extension,
 )
 
-
 __doctests__ = ['scrapy.utils.url']
 
 
@@ -81,6 +83,15 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
         self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
 
+    def test_url_has_any_extension(self):
+        deny_extensions = {'.' + e for e in arg_to_iter(IGNORED_EXTENSIONS)}
+        self.assertTrue(url_has_any_extension("http://www.example.com/archive.tar.gz", deny_extensions))
+        self.assertTrue(url_has_any_extension("http://www.example.com/page.doc", deny_extensions))
+        self.assertTrue(url_has_any_extension("http://www.example.com/page.pdf", deny_extensions))
+        self.assertFalse(url_has_any_extension("http://www.example.com/page.htm", deny_extensions))
+        self.assertFalse(url_has_any_extension("http://www.example.com/", deny_extensions))
+        self.assertFalse(url_has_any_extension("http://www.example.com/page.doc.html", deny_extensions))
+
 
 class AddHttpIfNoScheme(unittest.TestCase):