Repository: manuzhang/mkdocs-htmlproofer-plugin

Issue:
mailto: links generate 404 errors
If the document contains mailto: links the plugin returns 404 for each of them.

**To Reproduce**
create a mailto: link (for example `mailto:toto@toto.com`) in a source .md file and build your documentation.

It should simply ignore mailto links.

Discussion:
manuzhang: LGTM. Please fix styles issues in #55
kolibri91: - [ ] Merge `main` after #55, it's expected to work without conflicts by using default *ort*-strategy.
kolibri91: @manuzhang Did you consider to activate branch protection rule "up to date". This would ensure that the PRs contains latest changes.
manuzhang: It doesn't have to as long as there's no conflict.
kolibri91: I noticed in my daily work two common issues with branches which are not up to date before you merge.

It can happen that the ci dies not work correctly if main branch contains CI configuration updates. That would be the case here. Fortunately, I noticed the issues during development of this branch.

Secondly, auto-merge can break styling.

Patch:
diff --git a/htmlproofer/plugin.py b/htmlproofer/plugin.py
index 1405652..5420840 100644
--- a/htmlproofer/plugin.py
+++ b/htmlproofer/plugin.py
@@ -1,13 +1,14 @@
-from functools import lru_cache
+from functools import lru_cache, partial
 import os.path
 import pathlib
 import re
-import sys
 from typing import Dict, Optional, Set
+import urllib.parse
 import uuid
 
 from bs4 import BeautifulSoup, SoupStrainer
 from markdown.extensions.toc import slugify
+from mkdocs import utils
 from mkdocs.config import Config, config_options
 from mkdocs.exceptions import PluginError
 from mkdocs.plugins import BasePlugin
@@ -19,8 +20,8 @@ import urllib3
 URL_TIMEOUT = 10.0
 _URL_BOT_ID = f'Bot {uuid.uuid4()}'
 URL_HEADERS = {'User-Agent': _URL_BOT_ID, 'Accept-Language': '*'}
+NAME = "htmlproofer"
 
-EXTERNAL_URL_PATTERN = re.compile(r'https?://')
 MARKDOWN_ANCHOR_PATTERN = re.compile(r'([^#]+)(#(.+))?')
 HEADING_PATTERN = re.compile(r'\s*#+\s*(.*)')
 HTML_LINK_PATTERN = re.compile(r'.*<a id=\"(.*)\">.*')
@@ -35,6 +36,18 @@ ATTRLIST_PATTERN = re.compile(r'\{.*?\}')
 urllib3.disable_warnings()
 
 
+def log_info(msg, *args, **kwargs):
+    utils.log.info(f"{NAME}: {msg}", *args, **kwargs)
+
+
+def log_warning(msg, *args, **kwargs):
+    utils.log.warning(f"{NAME}: {msg}", *args, **kwargs)
+
+
+def log_error(msg, *args, **kwargs):
+    utils.log.error(f"{NAME}: {msg}", *args, **kwargs)
+
+
 class HtmlProoferPlugin(BasePlugin):
     files: Dict[str, File]
     invalid_links = False
@@ -54,6 +67,10 @@ class HtmlProoferPlugin(BasePlugin):
         self._session.headers.update(URL_HEADERS)
         self._session.max_redirects = 5
         self.files = {}
+        self.scheme_handlers = {
+            "http": partial(HtmlProoferPlugin.resolve_web_scheme, self),
+            "https": partial(HtmlProoferPlugin.resolve_web_scheme, self),
+        }
         super().__init__()
 
     def on_post_build(self, config: Config) -> None:
@@ -91,12 +108,20 @@ class HtmlProoferPlugin(BasePlugin):
                 if self.config['raise_error'] and is_error:
                     raise PluginError(error)
                 elif self.config['raise_error_after_finish'] and is_error and not self.invalid_links:
+                    log_error(error)
                     self.invalid_links = True
                 if is_error:
-                    print(error)
+                    log_warning(error)
+
+    def get_external_url(self, url, scheme, src_path):
+        try:
+            return self.scheme_handlers[scheme](url)
+        except KeyError:
+            log_info(f'Unknown url-scheme "{scheme}:" detected. "{url}" from "{src_path}" will not be checked.')
+        return 0
 
     @lru_cache(maxsize=1000)
-    def get_external_url(self, url: str) -> int:
+    def resolve_web_scheme(self, url: str) -> int:
         try:
             response = self._session.get(url, timeout=URL_TIMEOUT)
             return response.status_code
@@ -118,12 +143,13 @@ class HtmlProoferPlugin(BasePlugin):
         if any(pat.match(url) for pat in LOCAL_PATTERNS):
             return 0
 
-        if url.startswith('#'):
+        scheme, _, path, _, fragment = urllib.parse.urlsplit(url)
+        if scheme:
+            if self.config['validate_external_urls']:
+                return self.get_external_url(url, scheme, src_path)
+            return 0
+        if fragment and not path:
             return 0 if url[1:] in all_element_ids else 404
-        elif EXTERNAL_URL_PATTERN.match(url):
-            if not self.config['validate_external_urls']:
-                return 0
-            return self.get_external_url(url)
         elif not use_directory_urls:
             # use_directory_urls = True injects too many challenges for locating the correct target
             # Markdown file, so disable target anchor validation in this case. Examples include:
@@ -178,7 +204,7 @@ class HtmlProoferPlugin(BasePlugin):
         try:
             return files[search_path]
         except KeyError:
-            print(f"Warning: Unable to locate source file for: {url}", file=sys.stderr)
+            utils.log.warning(f"Unable to locate source file for: {url}")
             return None
 
     @staticmethod


Test patch:
diff --git a/tests/integration/docs/index.md b/tests/integration/docs/index.md
index 4d302a6..76ff389 100644
--- a/tests/integration/docs/index.md
+++ b/tests/integration/docs/index.md
@@ -81,3 +81,16 @@ This work is based on the [mkdocs-markdownextradata-plugin](https://github.com/r
 <a href="assets/hello-world.drawio.svg">![test](assets/hello-world.drawio.svg)</a>
 
 <a href="/assets/hello-world.drawio.svg">![test](/assets/hello-world.drawio.svg)</a>
+
+## External URLs
+
+External urls starting with `http` or `https` are checked.
+
+- <http://github.com>
+- <https://github.com>
+
+## Custom schemes
+
+Unkown schemes like `mailto` are ignored.
+
+- <mailto:info@github.com>
diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py
index dd8b220..e060d19 100644
--- a/tests/unit/test_plugin.py
+++ b/tests/unit/test_plugin.py
@@ -5,6 +5,7 @@ from mkdocs.config import Config
 from mkdocs.exceptions import PluginError
 from mkdocs.structure.files import File, Files
 from mkdocs.structure.pages import Page
+import mkdocs.utils
 import pytest
 from requests import Response
 
@@ -181,13 +182,58 @@ def test_get_url_status__same_page_anchor(plugin, empty_files):
         'https://extwebsite.com',
         'http://extwebsite.com',
         'https://website.net/path#anchor',
+        'mailto:toto@toto.com',
+        'steam://application',
+        'file://file',
     ),
 )
 def test_get_url_status__external(plugin, empty_files, url):
+    src_path = 'src/path.md'
+    scheme = url.split(":")[0]
+    expected_status = 200
+
     with patch.object(HtmlProoferPlugin, "get_external_url") as mock_get_ext_url:
-        mock_get_ext_url.return_value = 200
-        assert plugin.get_url_status(url, 'src/path.md', set(), empty_files, False) == 200
-    mock_get_ext_url.assert_called_once_with(url)
+        mock_get_ext_url.return_value = expected_status
+        status = plugin.get_url_status(url, src_path, set(), empty_files, False)
+
+    mock_get_ext_url.assert_called_once_with(url, scheme, src_path)
+    assert status == expected_status
+
+
+@pytest.mark.parametrize("scheme", ('http', 'https'))
+def test_get_external_url__web_scheme(scheme):
+    src_path = 'src/path.md'
+    url = f"{scheme}://path.html"
+    expected_status = 200
+
+    with patch.object(HtmlProoferPlugin, "resolve_web_scheme") as mock_resolve_web_scheme:
+        mock_resolve_web_scheme.return_value = expected_status
+        plugin = HtmlProoferPlugin()
+        plugin.load_config({})
+
+        status = plugin.get_external_url(url, scheme, src_path)
+
+    mock_resolve_web_scheme.assert_called_once_with(plugin, url)
+    assert status == expected_status
+
+
+@pytest.mark.parametrize("scheme", ('mailto', 'file', 'steam', 'abc'))
+def test_get_external_url__unknown_scheme(scheme):
+    src_path = 'src/path.md'
+    url = f"{scheme}://path.html"
+    expected_status = 0
+
+    with patch.object(HtmlProoferPlugin, "resolve_web_scheme") as mock_resolve_web_scheme:
+        mock_resolve_web_scheme.return_value = expected_status
+        plugin = HtmlProoferPlugin()
+        plugin.load_config({})
+
+        with patch.object(mkdocs.utils.log, "info") as mock_log_info:
+            status = plugin.get_external_url(url, scheme, src_path)
+
+    mock_log_info.assert_called_once()
+    mock_resolve_web_scheme.assert_not_called()
+    assert status == expected_status
 
 
 def test_get_url_status__local_page(plugin):