Repository: scrapy/w3lib

Issue:
It's not a good idead to parse HTML text using regular expressions
In [`w3lib.html`](https://github.com/scrapy/w3lib/blob/master/w3lib/html.py) regular expressions are used to parse HTML texts:

``` python
_ent_re = re.compile(r'&((?P<named>[a-z\d]+)|#(?P<dec>\d+)|#x(?P<hex>[a-f\d]+))(?P<semicolon>;?)', re.IGNORECASE)
_tag_re = re.compile(r'<[a-zA-Z\/!].*?>', re.DOTALL)
_baseurl_re = re.compile(six.u(r'<base\s[^>]*href\s*=\s*[\"\']\s*([^\"\'\s]+)\s*[\"\']'), re.I)
_meta_refresh_re = re.compile(six.u(r'<meta\s[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>(\d*\.)?\d+)\s*;\s*url=\s*(?P<url>.*?)(?P=quote)'), re.DOTALL | re.IGNORECASE)
_cdata_re = re.compile(r'((?P<cdata_s><!\[CDATA\[)(?P<cdata_d>.*?)(?P<cdata_e>\]\]>))', re.DOTALL)
```

However this is definitely incorrect when it involves commented contents, e.g.

``` python
>>> from w3lib import html
>>> html.get_base_url("""<!-- <base href="http://example.com/" /> -->""")
'http://example.com/'
```

Introducing "heavier" utilities like `lxml` would solve this issue easily, but that might be an awful idea as `w3lib` aims to be lightweight & fast.  
Or maybe we could implement some quick parser merely for eliminating the commented parts.

Any ideas?


Comments:

Patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 50f3226..ee3f8e5 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -35,10 +35,10 @@ jobs:
             TOXENV: typing
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 787f56a..f694f42 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python 3.9
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
 
diff --git a/README.rst b/README.rst
index 13745f8..5664183 100644
--- a/README.rst
+++ b/README.rst
@@ -27,7 +27,7 @@ This is a Python library of web-related functions, such as:
 Requirements
 ============
 
-Python 3.6+
+Python 3.7+
 
 Install
 =======
diff --git a/docs/index.rst b/docs/index.rst
index bd14188..aa1c851 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -28,7 +28,7 @@ Modules
 Requirements
 ============
 
-Python 3.6+
+Python 3.7+
 
 Install
 =======
diff --git a/setup.py b/setup.py
index fe1e5fe..7fdb3b1 100644
--- a/setup.py
+++ b/setup.py
@@ -16,14 +16,13 @@ setup(
     include_package_data=True,
     zip_safe=False,
     platforms=["Any"],
-    python_requires=">=3.6",
+    python_requires=">=3.7",
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
diff --git a/w3lib/html.py b/w3lib/html.py
index a4be054..a31d42b 100644
--- a/w3lib/html.py
+++ b/w3lib/html.py
@@ -311,7 +311,7 @@ def get_base_url(
 
     """
 
-    utext = to_unicode(text, encoding)
+    utext: str = remove_comments(text, encoding=encoding)
     m = _baseurl_re.search(utext)
     if m:
         return urljoin(


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 6bd16ed..031ee7b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -14,13 +14,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11.0-rc.2", "pypy3"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.7"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
diff --git a/tests/test_html.py b/tests/test_html.py
index d4861ba..1e637b0 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -372,6 +372,30 @@ class GetBaseUrlTest(unittest.TestCase):
             get_base_url(text, baseurl.encode("ascii")), "http://example.org/something"
         )
 
+    def test_base_url_in_comment(self):
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/> -->"""), ""
+        )
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/>"""), ""
+        )
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/> --"""), ""
+        )
+        self.assertEqual(
+            get_base_url(
+                """<!-- <!--  <base href="http://example.com/"/> -- -->  <base href="http://example_2.com/"/> """
+            ),
+            "http://example_2.com/",
+        )
+
+        self.assertEqual(
+            get_base_url(
+                """<!-- <base href="http://example.com/"/> --> <!-- <base href="http://example_2.com/"/> --> <base href="http://example_3.com/"/>"""
+            ),
+            "http://example_3.com/",
+        )
+
     def test_relative_url_with_absolute_path(self):
         baseurl = "https://example.org"
         text = """\