Repository: scrapy/w3lib

Issue:
It's not a good idead to parse HTML text using regular expressions
In [`w3lib.html`](https://github.com/scrapy/w3lib/blob/master/w3lib/html.py) regular expressions are used to parse HTML texts:

``` python
_ent_re = re.compile(r'&((?P<named>[a-z\d]+)|#(?P<dec>\d+)|#x(?P<hex>[a-f\d]+))(?P<semicolon>;?)', re.IGNORECASE)
_tag_re = re.compile(r'<[a-zA-Z\/!].*?>', re.DOTALL)
_baseurl_re = re.compile(six.u(r'<base\s[^>]*href\s*=\s*[\"\']\s*([^\"\'\s]+)\s*[\"\']'), re.I)
_meta_refresh_re = re.compile(six.u(r'<meta\s[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>(\d*\.)?\d+)\s*;\s*url=\s*(?P<url>.*?)(?P=quote)'), re.DOTALL | re.IGNORECASE)
_cdata_re = re.compile(r'((?P<cdata_s><!\[CDATA\[)(?P<cdata_d>.*?)(?P<cdata_e>\]\]>))', re.DOTALL)
```

However this is definitely incorrect when it involves commented contents, e.g.

``` python
>>> from w3lib import html
>>> html.get_base_url("""<!-- <base href="http://example.com/" /> -->""")
'http://example.com/'
```

Introducing "heavier" utilities like `lxml` would solve this issue easily, but that might be an awful idea as `w3lib` aims to be lightweight & fast.  
Or maybe we could implement some quick parser merely for eliminating the commented parts.

Any ideas?


Discussion:
redapple: Can you add tests for this?
Can you provide example websites showing this issue?

Patch:
diff --git a/w3lib/html.py b/w3lib/html.py
index a4be054..a31d42b 100644
--- a/w3lib/html.py
+++ b/w3lib/html.py
@@ -311,7 +311,7 @@ def get_base_url(
 
     """
 
-    utext = to_unicode(text, encoding)
+    utext: str = remove_comments(text, encoding=encoding)
     m = _baseurl_re.search(utext)
     if m:
         return urljoin(


Test patch:
diff --git a/tests/test_html.py b/tests/test_html.py
index d4861ba..1e637b0 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -372,6 +372,30 @@ class GetBaseUrlTest(unittest.TestCase):
             get_base_url(text, baseurl.encode("ascii")), "http://example.org/something"
         )
 
+    def test_base_url_in_comment(self):
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/> -->"""), ""
+        )
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/>"""), ""
+        )
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/> --"""), ""
+        )
+        self.assertEqual(
+            get_base_url(
+                """<!-- <!--  <base href="http://example.com/"/> -- -->  <base href="http://example_2.com/"/> """
+            ),
+            "http://example_2.com/",
+        )
+
+        self.assertEqual(
+            get_base_url(
+                """<!-- <base href="http://example.com/"/> --> <!-- <base href="http://example_2.com/"/> --> <base href="http://example_3.com/"/>"""
+            ),
+            "http://example_3.com/",
+        )
+
     def test_relative_url_with_absolute_path(self):
         baseurl = "https://example.org"
         text = """\