Repository: scrapy/scrapy
Pull request #5881 (2023-03-30 13:28:42)

Patched LxmlParserLinkExtractor
# Issue
When processing links, in certain edge-cases I have noticed crawlers crashing with the following/similar error, when all other links are parsed successfully:

```
Traceback (most recent call last):
  File "/usr/local/lib/python3.11/site-packages/scrapy/utils/defer.py", line 254, in aiter_errback
    yield await it.__anext__()
          ^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/scrapy/utils/python.py", line 366, in __anext__
    return await self.data.__anext__()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/scrapy/utils/python.py", line 347, in _async_chain
    async for o in as_async_generator(it):
  File "/usr/local/lib/python3.11/site-packages/scrapy/utils/asyncgen.py", line 14, in as_async_generator
    async for r in it:
  File "/usr/local/lib/python3.11/site-packages/scrapy/utils/python.py", line 366, in __anext__
    return await self.data.__anext__()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/scrapy/utils/python.py", line 347, in _async_chain
    async for o in as_async_generator(it):
  File "/usr/local/lib/python3.11/site-packages/scrapy/utils/asyncgen.py", line 14, in as_async_generator
    async for r in it:
  File "/usr/local/lib/python3.11/site-packages/scrapy/core/spidermw.py", line 90, in process_async
    async for r in iterable:
  File "/usr/local/lib/python3.11/site-packages/scrapy/spidermiddlewares/depth.py", line 36, in process_spider_output_async
    async for r in result or ():
  File "/usr/local/lib/python3.11/site-packages/scrapy/core/spidermw.py", line 90, in process_async
    async for r in iterable:
  File "/usr/local/lib/python3.11/site-packages/scrapy/spidermiddlewares/offsite.py", line 32, in process_spider_output_async
    async for r in result or ():
  File "/usr/local/lib/python3.11/site-packages/scrapy/core/spidermw.py", line 90, in process_async
    async for r in iterable:
  File "/usr/local/lib/python3.11/site-packages/scrapy/spidermiddlewares/referer.py", line 339, in process_spider_output_async
    async for r in result or ():
  File "/usr/local/lib/python3.11/site-packages/scrapy/core/spidermw.py", line 90, in process_async
    async for r in iterable:
  File "/usr/local/lib/python3.11/site-packages/scrapy/spidermiddlewares/urllength.py", line 31, in process_spider_output_async
    async for r in result or ():
  File "/usr/local/lib/python3.11/site-packages/scrapy/core/spidermw.py", line 90, in process_async
    async for r in iterable:
  File "/usr/local/lib/python3.11/site-packages/scrapy/spiders/crawl.py", line 125, in _parse_response
    for request_or_item in self._requests_to_follow(response):
  File "/usr/local/lib/python3.11/site-packages/scrapy/spiders/crawl.py", line 98, in _requests_to_follow
    links = [lnk for lnk in rule.link_extractor.extract_links(response)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/scrapy/linkextractors/lxmlhtml.py", line 162, in extract_links
    links = self._extract_links(doc, response.url, response.encoding, base_url)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/scrapy/linkextractors/__init__.py", line 132, in _extract_links
    return self.link_extractor._extract_links(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/scrapy/linkextractors/lxmlhtml.py", line 76, in _extract_links
    url = safe_url_string(url, encoding=response_encoding)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/w3lib/url.py", line 148, in safe_url_string
    parts.port,
    ^^^^^^^^^^
  File "/usr/local/lib/python3.11/urllib/parse.py", line 173, in port
    raise ValueError(f"Port could not be cast to integer value as {port!r}")
ValueError: Port could not be cast to integer value as ' https:'
```
I believe the cases in question are caused by bad formatting in the websites which results in two URLs back-to-back with no separation, causing the `LinkExtractor` to attempt to parse the seconds `https:` as a port, which fails as it can't be cast to `int`.  

## The Fix

The fix I have applied simply wraps the `url = safe_url_string(url, encoding=response_encoding)` call in a try/except case that looks for the `ValueError`, so that these links/formatting issues can be ignored and do not cause crawlers to crash.

As I can't pinpoint where the error happens on the pages I've noticed it, I have not been able to include tests, but after applying the patch locally and running crawlers, all functionality looks normal (with the added bonus of no unexpected crashes 👍🏻)



Patch:
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index dd8dcdf7c..23cbd0116 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -1,6 +1,7 @@
 """
 Link extractor based on lxml.html
 """
+import logging
 import operator
 from functools import partial
 from urllib.parse import urljoin, urlparse
@@ -23,6 +24,8 @@ from scrapy.utils.python import unique as unique_list
 from scrapy.utils.response import get_base_url
 from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain
 
+logger = logging.getLogger(__name__)
+
 # from lxml/src/lxml/html/__init__.py
 XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
 
@@ -88,7 +91,12 @@ class LxmlParserLinkExtractor:
                 url = self.process_attr(attr_val)
                 if url is None:
                     continue
-            url = safe_url_string(url, encoding=response_encoding)
+            try:
+                url = safe_url_string(url, encoding=response_encoding)
+            except ValueError:
+                logger.debug(f"Skipping extraction of link with bad URL {url!r}")
+                continue
+
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(
diff --git a/tox.ini b/tox.ini
index 5a9d9cf29..873e7662b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -101,6 +101,7 @@ install_command =
     python -I -m pip install {opts} {packages}
 
 [testenv:pinned]
+basepython = python3.7
 deps =
     {[pinned]deps}
     PyDispatcher==2.0.5


Test patch:
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index f663013ba..e1ec19601 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -2,6 +2,10 @@ import pickle
 import re
 import unittest
 
+from packaging.version import Version
+from pytest import mark
+from w3lib import __version__ as w3lib_version
+
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.link import Link
 from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
@@ -815,3 +819,34 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
 
     def test_restrict_xpaths_with_html_entities(self):
         super().test_restrict_xpaths_with_html_entities()
+
+    @mark.skipif(
+        Version(w3lib_version) < Version("2.0.0"),
+        reason=(
+            "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain "
+            "about an invalid port value."
+        ),
+    )
+    def test_skip_bad_links(self):
+        html = b"""
+        <a href="http://example.org:non-port">Why would you do this?</a>
+        <a href="http://example.org/item2.html">Good Link</a>
+        <a href="http://example.org/item3.html">Good Link 2</a>
+        """
+        response = HtmlResponse("http://example.org/index.html", body=html)
+        lx = self.extractor_cls()
+        self.assertEqual(
+            [link for link in lx.extract_links(response)],
+            [
+                Link(
+                    url="http://example.org/item2.html",
+                    text="Good Link",
+                    nofollow=False,
+                ),
+                Link(
+                    url="http://example.org/item3.html",
+                    text="Good Link 2",
+                    nofollow=False,
+                ),
+            ],
+        )