Repository: pypa/pip

Issue:
Ignore leading whitespace when checking for HTML doctype
<!---
Thank you for your soon to be pull request. Before you submit this, please
double check to make sure that you've added a news file fragment. In pip we
generate our NEWS.rst from multiple news fragment files, and all pull requests
require either a news file fragment or a marker to indicate they don't require
one.

To read more about adding a news file fragment for your PR, please check out
our documentation at: https://pip.pypa.io/en/latest/development/contributing/#news-entries
-->

Ref https://github.com/pypa/pip/issues/10825#issuecomment-1025534650


Discussion:
I'm not sure about the CI failures. They're getting `AttributeError: 'int' object has no attribute 'startswith'` from within setuptools.
Hmm, I read the [HTML5 spec](https://html.spec.whatwg.org/multipage/syntax.html#the-doctype) and it seems we are still missing a lot of logic. This PR stands on its own so we should still merge it (unless you want to tackle them in this PR), but hopefully someone can come in and fix those other issues.
> I'm not sure about the CI failures. They're getting `AttributeError: 'int' object has no attribute 'startswith'` from within setuptools.

Yes, seems tests are broken on Ubuntu at the moment, I am getting the same error for something completely unrelated: https://github.com/pypa/pip/runs/5010487348?check_suite_focus=true
I see @domdfcoding has filed https://github.com/pypa/setuptools/issues/3063 already! :)
> hopefully someone can come in and fix those other issues.

I'm willing to take a look at this (I was intending to, in fact) but my time till the end of February is somewhat limited, so if someone else wants to get to it first that's fine.
In https://github.com/pypa/pip/issues/10868, I'm proposing we drop this fallback I added in the spirit of expidiecy yesterday -- I think there's a much better behaviour here that I didn't think of in the heat of the moment yesterday.

Resolving patch:
diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py
--- a/src/pip/_internal/exceptions.py
+++ b/src/pip/_internal/exceptions.py
@@ -181,6 +181,30 @@ class UninstallationError(PipError):
     """General exception during uninstallation"""
 
 
+class BadHTMLDoctypeDeclaration(DiagnosticPipError):
+    reference = "bad-index-doctype"
+
+    def __init__(self, *, url: str) -> None:
+        super().__init__(
+            kind="warning",
+            message=(
+                "The package index page being used does not have a proper HTML "
+                "doctype declaration."
+            ),
+            context=f"Problematic URL: {escape(url)}",
+            note_stmt="This is an issue with the page at the URL mentioned above.",
+            hint_stmt=(
+                "You might need to reach out to the owner of that package index, "
+                "to get this fixed. "
+                "See https://github.com/pypa/pip/issues/10825 for context."
+            ),
+        )
+
+
+class MissingHTMLDoctypeDeclaration(BadHTMLDoctypeDeclaration):
+    reference = "missing-index-doctype"
+
+
 class MissingPyProjectBuildRequires(DiagnosticPipError):
     """Raised when pyproject.toml has `build-system`, but no `build-system.requires`."""
 
diff --git a/src/pip/_internal/index/collector.py b/src/pip/_internal/index/collector.py
--- a/src/pip/_internal/index/collector.py
+++ b/src/pip/_internal/index/collector.py
@@ -16,7 +16,6 @@
 from optparse import Values
 from typing import (
     TYPE_CHECKING,
-    Any,
     Callable,
     Dict,
     Iterable,
@@ -33,12 +32,15 @@
 from pip._vendor.requests import Response
 from pip._vendor.requests.exceptions import RetryError, SSLError
 
-from pip._internal.exceptions import NetworkConnectionError
+from pip._internal.exceptions import (
+    BadHTMLDoctypeDeclaration,
+    MissingHTMLDoctypeDeclaration,
+    NetworkConnectionError,
+)
 from pip._internal.models.link import Link
 from pip._internal.models.search_scope import SearchScope
 from pip._internal.network.session import PipSession
 from pip._internal.network.utils import raise_for_status
-from pip._internal.utils.deprecation import deprecated
 from pip._internal.utils.filetypes import is_archive_file
 from pip._internal.utils.misc import pairwise, redact_auth_from_url
 from pip._internal.vcs import vcs
@@ -343,34 +345,13 @@ def parse_links(page: "HTMLPage", use_deprecated_html5lib: bool) -> Iterable[Lin
     """
     Parse an HTML document, and yield its anchor elements as Link objects.
     """
-    encoding = page.encoding or "utf-8"
-
-    # Check if the page starts with a valid doctype, to decide whether to use
-    # http.parser or (deprecated) html5lib for parsing -- unless explicitly
-    # requested to use html5lib.
-    if not use_deprecated_html5lib:
-        expected_doctype = "<!doctype html>".encode(encoding)
-        actual_start = page.content[: len(expected_doctype)]
-        if actual_start.decode(encoding).lower() != "<!doctype html>":
-            deprecated(
-                reason=(
-                    f"The HTML index page being used ({page.url}) is not a proper "
-                    "HTML 5 document. This is in violation of PEP 503 which requires "
-                    "these pages to be well-formed HTML 5 documents. Please reach out "
-                    "to the owners of this index page, and ask them to update this "
-                    "index page to a valid HTML 5 document."
-                ),
-                replacement=None,
-                gone_in="22.2",
-                issue=10825,
-            )
-            use_deprecated_html5lib = True
 
     if use_deprecated_html5lib:
         yield from _parse_links_html5lib(page)
         return
 
-    parser = HTMLLinkParser()
+    parser = HTMLLinkParser(page.url)
+    encoding = page.encoding or "utf-8"
     parser.feed(page.content.decode(encoding))
 
     url = page.url
@@ -418,20 +399,34 @@ class HTMLLinkParser(HTMLParser):
     elements' attributes.
     """
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        super().__init__(*args, **kwargs)
-        self._seen_decl = False
+    def __init__(self, url: str) -> None:
+        super().__init__(convert_charrefs=True)
+        self._dealt_with_doctype_issues = False
+
+        self.url: str = url
         self.base_url: Optional[str] = None
         self.anchors: List[Dict[str, Optional[str]]] = []
 
     def handle_decl(self, decl: str) -> None:
-        if decl.lower() != "doctype html":
-            self._raise_error()
-        self._seen_decl = True
+        self._dealt_with_doctype_issues = True
+        match = re.match(
+            r"""doctype\s+html\s*(?:SYSTEM\s+(["'])about:legacy-compat\1)?\s*$""",
+            decl,
+            re.IGNORECASE,
+        )
+        if match is None:
+            logger.warning(
+                "[present-diagnostic] %s",
+                BadHTMLDoctypeDeclaration(url=self.url),
+            )
 
     def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None:
-        if not self._seen_decl:
-            self._raise_error()
+        if not self._dealt_with_doctype_issues:
+            logger.warning(
+                "[present-diagnostic] %s",
+                MissingHTMLDoctypeDeclaration(url=self.url),
+            )
+            self._dealt_with_doctype_issues = True
 
         if tag == "base" and self.base_url is None:
             href = self.get_href(attrs)
@@ -446,14 +441,6 @@ def get_href(self, attrs: List[Tuple[str, Optional[str]]]) -> Optional[str]:
                 return value
         return None
 
-    def _raise_error(self) -> None:
-        raise ValueError(
-            "HTML doctype missing or incorrect. Expected <!DOCTYPE html>.\n\n"
-            "If you believe this error to be incorrect, try passing the "
-            "command line option --use-deprecated=html5lib and please leave "
-            "a comment on the pip issue at https://github.com/pypa/pip/issues/10825."
-        )
-
 
 def _handle_get_page_fail(
     link: Link,


Test patch:
diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py
--- a/tests/unit/test_collector.py
+++ b/tests/unit/test_collector.py
@@ -551,21 +551,42 @@ def test_parse_link_handles_deprecated_usage_properly() -> None:
     assert "pkg1-2.0" in parsed_links[1].url
 
 
-@mock.patch("pip._internal.index.collector.deprecated")
-def test_parse_links_presents_deprecation_warning_on_non_html5_page(
-    mock_deprecated: mock.Mock,
+def test_parse_links_presents_warning_on_missing_doctype(
+    caplog: pytest.LogCaptureFixture,
 ) -> None:
     html = b'<a href="/pkg1-1.0.tar.gz"></a><a href="/pkg1-2.0.tar.gz"></a>'
     url = "https://example.com/simple/"
     page = HTMLPage(html, encoding=None, url=url, cache_link_parsing=False)
 
-    parsed_links = list(parse_links(page, use_deprecated_html5lib=False))
+    with caplog.at_level(logging.WARN):
+        parsed_links = list(parse_links(page, use_deprecated_html5lib=False))
 
     assert len(parsed_links) == 2, parsed_links
     assert "pkg1-1.0" in parsed_links[0].url
     assert "pkg1-2.0" in parsed_links[1].url
 
-    mock_deprecated.assert_called_once()
+    assert len(caplog.records) == 1
+
+
+def test_parse_links_presents_warning_on_html4_doctype(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    html = (
+        b'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" '
+        b'"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">'
+        b'<a href="/pkg1-1.0.tar.gz"></a><a href="/pkg1-2.0.tar.gz"></a>'
+    )
+    url = "https://example.com/simple/"
+    page = HTMLPage(html, encoding=None, url=url, cache_link_parsing=False)
+
+    with caplog.at_level(logging.WARN):
+        parsed_links = list(parse_links(page, use_deprecated_html5lib=False))
+
+    assert len(parsed_links) == 2, parsed_links
+    assert "pkg1-1.0" in parsed_links[0].url
+    assert "pkg1-2.0" in parsed_links[1].url
+
+    assert len(caplog.records) == 1
 
 
 @mock.patch("pip._internal.index.collector.raise_for_status")