Repository: andreoliwa/nitpick (language: python)

Issue:
feat: Support custom protocols for styles
## Proposed changes
When it is done, this should allow URLs like `github://`, `gh://`, and others.

## Checklist

- [x] Read the [contribution guidelines](https://nitpick.readthedocs.io/en/latest/contributing.html)
- [x] Run `make` locally before pushing commits
- [ ] Add tests for:
  - [ ] API
  - [ ] CLI
  - [ ] `flake8` plugin (normal mode)
  - [ ] `flake8` plugin (offline mode)
- [ ] Write documentation when there's new API, functionality, etc.:
  - [ ] On `README.md`
  - [ ] On `docs/*.rst` files



Discussion / hints:

Fix patch:
diff --git a/Makefile b/Makefile
index af18f5bc..e78dc180 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 .DEFAULT_GOAL := build
 .PHONY: Makefile
 
-SRC := $(shell find -f docs src -type f -iname '*.py')
+SRC := $(shell find docs src -type f -a -iname '*.py')
 DOCS := docs/*.rst *.rst *.md
 STYLES := $(shell find styles -type f)
 TESTS := $(shell find tests -type f -iname '*.py')
diff --git a/docs/source/nitpick.style.fetchers.base.rst b/docs/source/nitpick.style.fetchers.base.rst
new file mode 100644
index 00000000..dbf21398
--- /dev/null
+++ b/docs/source/nitpick.style.fetchers.base.rst
@@ -0,0 +1,7 @@
+nitpick.style.fetchers.base module
+==================================
+
+.. automodule:: nitpick.style.fetchers.base
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/nitpick.style.fetchers.file.rst b/docs/source/nitpick.style.fetchers.file.rst
new file mode 100644
index 00000000..84776487
--- /dev/null
+++ b/docs/source/nitpick.style.fetchers.file.rst
@@ -0,0 +1,7 @@
+nitpick.style.fetchers.file module
+==================================
+
+.. automodule:: nitpick.style.fetchers.file
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/nitpick.style.fetchers.github.rst b/docs/source/nitpick.style.fetchers.github.rst
new file mode 100644
index 00000000..d9fb8082
--- /dev/null
+++ b/docs/source/nitpick.style.fetchers.github.rst
@@ -0,0 +1,7 @@
+nitpick.style.fetchers.github module
+====================================
+
+.. automodule:: nitpick.style.fetchers.github
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/nitpick.style.fetchers.http.rst b/docs/source/nitpick.style.fetchers.http.rst
new file mode 100644
index 00000000..1c72aeff
--- /dev/null
+++ b/docs/source/nitpick.style.fetchers.http.rst
@@ -0,0 +1,7 @@
+nitpick.style.fetchers.http module
+==================================
+
+.. automodule:: nitpick.style.fetchers.http
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/nitpick.style.fetchers.rst b/docs/source/nitpick.style.fetchers.rst
index 887fe07a..665fa15e 100644
--- a/docs/source/nitpick.style.fetchers.rst
+++ b/docs/source/nitpick.style.fetchers.rst
@@ -1,7 +1,18 @@
-nitpick.style.fetchers module
-=============================
+nitpick.style.fetchers package
+==============================
 
 .. automodule:: nitpick.style.fetchers
    :members:
    :undoc-members:
    :show-inheritance:
+
+Submodules
+----------
+
+.. toctree::
+   :maxdepth: 4
+
+   nitpick.style.fetchers.base
+   nitpick.style.fetchers.file
+   nitpick.style.fetchers.github
+   nitpick.style.fetchers.http
diff --git a/docs/source/nitpick.style.rst b/docs/source/nitpick.style.rst
index cbd8f87c..acb0eb1b 100644
--- a/docs/source/nitpick.style.rst
+++ b/docs/source/nitpick.style.rst
@@ -6,6 +6,14 @@ nitpick.style package
    :undoc-members:
    :show-inheritance:
 
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   nitpick.style.fetchers
+
 Submodules
 ----------
 
@@ -15,4 +23,3 @@ Submodules
    nitpick.style.cache
    nitpick.style.config
    nitpick.style.core
-   nitpick.style.fetchers
diff --git a/src/nitpick/style/fetchers.py b/src/nitpick/style/fetchers.py
deleted file mode 100644
index 856e8212..00000000
--- a/src/nitpick/style/fetchers.py
+++ /dev/null
@@ -1,184 +0,0 @@
-"""Style fetchers with protocol support."""
-from dataclasses import dataclass, field
-from datetime import datetime
-from functools import lru_cache
-from pathlib import Path
-from typing import Dict, Optional, Tuple
-from urllib.parse import urlparse
-
-import click
-import requests
-from cachy import CacheManager, Repository
-from loguru import logger
-from requests import Session
-from slugify import slugify
-
-from nitpick.enums import CachingEnum, OptionEnum
-from nitpick.generic import is_url
-from nitpick.style.cache import parse_cache_option
-
-StyleInfo = Tuple[Optional[Path], str]
-FetchersType = Dict[str, "StyleFetcher"]
-
-# pylint: disable=too-few-public-methods
-
-
-@dataclass(repr=True)
-class StyleFetcherManager:
-    """Manager that controls which fetcher to be used given a protocol."""
-
-    offline: bool
-    cache_dir: str
-    cache_option: str
-
-    cache_repository: Repository = field(init=False)
-    fetchers: FetchersType = field(init=False)
-
-    def __post_init__(self):
-        """Initialize dependant properties."""
-        self.cache_repository = CacheManager({"stores": {"file": {"driver": "file", "path": self.cache_dir}}}).store()
-        self.fetchers = _get_fetchers(self.cache_repository, self.cache_option)
-
-    def fetch(self, url) -> StyleInfo:
-        """Determine which fetcher to be used and fetch from it."""
-        scheme = self._get_scheme(url)
-        try:
-            fetcher = self.fetchers[scheme]
-        except KeyError as exc:
-            raise RuntimeError(f"protocol {scheme} not supported") from exc
-
-        if self.offline and fetcher.requires_connection:
-            return None, ""
-
-        return fetcher.fetch(url)
-
-    @staticmethod
-    def _get_scheme(url: str) -> str:
-        r"""Get a scheme from an URL or a file.
-
-        >>> StyleFetcherManager._get_scheme("/abc")
-        'file'
-        >>> StyleFetcherManager._get_scheme("file:///abc")
-        'file'
-        >>> StyleFetcherManager._get_scheme(r"c:\abc")
-        'file'
-        >>> StyleFetcherManager._get_scheme("c:/abc")
-        'file'
-        >>> StyleFetcherManager._get_scheme("http://server.com/abc")
-        'http'
-        """
-        if is_url(url):
-            parsed_url = urlparse(url)
-            return parsed_url.scheme
-
-        return "file"
-
-
-@lru_cache(maxsize=1)
-def _requests_session() -> Session:
-    session = Session()
-    return session
-
-
-def _get_fetchers(cache_repository, cache_option) -> FetchersType:
-    def _factory(klass):
-        return klass(cache_repository, cache_option)
-
-    file_fetcher = _factory(FileFetcher)
-    http_fetcher = _factory(HttpFetcher)
-
-    return {
-        "file": file_fetcher,
-        "": file_fetcher,
-        "http": http_fetcher,
-        "https": http_fetcher,
-    }
-
-
-@dataclass(repr=True)
-class StyleFetcher:
-    """Base class of all fetchers, it encapsulate get/fetch from cache."""
-
-    cache_manager: CacheManager
-    cache_option: str
-
-    requires_connection = False
-
-    def fetch(self, url) -> StyleInfo:
-        """Fetch a style form cache or from a specific fetcher."""
-        caching, caching_delta = parse_cache_option(self.cache_option)
-        path = self._get_output_path(url)
-        cached = self._get_from_cache(caching, url)
-        if cached:
-            return path, cached
-
-        contents = self._do_fetch(url)
-        if not contents:
-            return None, ""
-
-        self._save_to_cache(caching, caching_delta, url, contents)
-        return path, contents
-
-    @staticmethod
-    def _get_output_path(url) -> Path:
-        if is_url(url):
-            return Path(slugify(url))
-
-        return Path(url)
-
-    def _get_from_cache(self, caching, url):
-        if caching == CachingEnum.NEVER:
-            return None
-
-        cached_value = self.cache_manager.get(url)
-        if cached_value is not None:
-            logger.debug(f"Using cached value for URL {url}")
-            return cached_value
-
-        return None
-
-    def _do_fetch(self, url):
-        raise NotImplementedError()
-
-    def _save_to_cache(self, caching, caching_delta, url, contents):
-        if caching == CachingEnum.FOREVER:
-            logger.debug(f"Caching forever the contents of {url}")
-            self.cache_manager.forever(str(url), contents)
-        elif caching == CachingEnum.EXPIRES:
-            future = datetime.now() + caching_delta
-            logger.debug(f"Caching the contents of {url} to expire in {future}")
-            self.cache_manager.put(str(url), contents, future)
-
-
-class HttpFetcher(StyleFetcher):
-    """Fetch a style from an http/https server."""
-
-    requires_connection = True
-
-    def _do_fetch(self, url) -> str:
-        try:
-            contents = self._download(url)
-        except requests.ConnectionError as err:
-            logger.exception(f"Request failed with {err}")
-            click.secho(
-                "Your network is unreachable. Fix your connection or use"
-                f" {OptionEnum.OFFLINE.as_flake8_flag()} / {OptionEnum.OFFLINE.as_envvar()}=1",
-                fg="red",
-                err=True,
-            )
-            return ""
-        return contents
-
-    @staticmethod
-    def _download(url) -> str:
-        response = _requests_session().get(url)
-        response.raise_for_status()
-        return response.text
-
-
-class FileFetcher(StyleFetcher):
-    """Fetch a style from a local file."""
-
-    def _do_fetch(self, url):
-        file_path = Path(url).expanduser()
-        return file_path.read_text()
diff --git a/src/nitpick/style/fetchers/__init__.py b/src/nitpick/style/fetchers/__init__.py
new file mode 100644
index 00000000..016f63dd
--- /dev/null
+++ b/src/nitpick/style/fetchers/__init__.py
@@ -0,0 +1,94 @@
+"""Style fetchers with protocol support."""
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Tuple
+from urllib.parse import urlparse
+
+from cachy import CacheManager, Repository
+from requests import Session
+
+from nitpick.generic import is_url
+
+if TYPE_CHECKING:
+    from nitpick.style.fetchers.base import FetchersType
+
+StyleInfo = Tuple[Optional[Path], str]
+
+
+# pylint: disable=too-few-public-methods
+
+
+@dataclass(repr=True)
+class StyleFetcherManager:
+    """Manager that controls which fetcher to be used given a protocol."""
+
+    offline: bool
+    cache_dir: str
+    cache_option: str
+
+    cache_repository: Repository = field(init=False)
+    fetchers: "FetchersType" = field(init=False)
+
+    def __post_init__(self):
+        """Initialize dependant properties."""
+        self.cache_repository = CacheManager({"stores": {"file": {"driver": "file", "path": self.cache_dir}}}).store()
+        self.fetchers = _get_fetchers(self.cache_repository, self.cache_option)
+
+    def fetch(self, url) -> StyleInfo:
+        """Determine which fetcher to be used and fetch from it."""
+        scheme = self._get_scheme(url)
+        try:
+            fetcher = self.fetchers[scheme]
+        except KeyError as exc:
+            raise RuntimeError(f"protocol {scheme} not supported") from exc
+
+        if self.offline and fetcher.requires_connection:
+            return None, ""
+
+        return fetcher.fetch(url)
+
+    @staticmethod
+    def _get_scheme(url: str) -> str:
+        r"""Get a scheme from an URL or a file.
+
+        >>> StyleFetcherManager._get_scheme("/abc")
+        'file'
+        >>> StyleFetcherManager._get_scheme("file:///abc")
+        'file'
+        >>> StyleFetcherManager._get_scheme(r"c:\abc")
+        'file'
+        >>> StyleFetcherManager._get_scheme("c:/abc")
+        'file'
+        >>> StyleFetcherManager._get_scheme("http://server.com/abc")
+        'http'
+        """
+        if is_url(url):
+            parsed_url = urlparse(url)
+            return parsed_url.scheme
+
+        return "file"
+
+
+def _requests_session() -> Session:
+    session = Session()
+    return session
+
+
+def _get_fetchers(cache_repository, cache_option) -> "FetchersType":
+    # pylint: disable=import-outside-toplevel
+    from nitpick.style.fetchers.file import FileFetcher
+    from nitpick.style.fetchers.github import GitHubFetcher
+    from nitpick.style.fetchers.http import HttpFetcher
+
+    def _factory(klass):
+        return klass(cache_repository, cache_option)
+
+    fetchers = (_factory(FileFetcher), _factory(HttpFetcher), _factory(GitHubFetcher))
+    pairs = _fetchers_to_pairs(fetchers)
+    return dict(pairs)
+
+
+def _fetchers_to_pairs(fetchers):
+    for fetcher in fetchers:
+        for protocol in fetcher.protocols:
+            yield protocol, fetcher
diff --git a/src/nitpick/style/fetchers/base.py b/src/nitpick/style/fetchers/base.py
new file mode 100644
index 00000000..507caeda
--- /dev/null
+++ b/src/nitpick/style/fetchers/base.py
@@ -0,0 +1,73 @@
+"""Base class for fetchers that wrap inner fetchers with caching ability."""
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Tuple
+
+from cachy import CacheManager
+from loguru import logger
+from slugify import slugify
+
+from nitpick.enums import CachingEnum
+from nitpick.generic import is_url
+from nitpick.style import parse_cache_option
+from nitpick.style.fetchers import StyleInfo
+
+
+@dataclass(repr=True)
+class StyleFetcher:
+    """Base class of all fetchers, it encapsulate get/fetch from cache."""
+
+    cache_manager: CacheManager
+    cache_option: str
+
+    requires_connection = False
+    protocols: Tuple[str, ...] = ()
+
+    def fetch(self, url) -> StyleInfo:
+        """Fetch a style form cache or from a specific fetcher."""
+        caching, caching_delta = parse_cache_option(self.cache_option)
+        path = self._get_output_path(url)
+        cached = self._get_from_cache(caching, url)
+        if cached:
+            return path, cached
+
+        contents = self._do_fetch(url)
+        if not contents:
+            return None, ""
+
+        self._save_to_cache(caching, caching_delta, url, contents)
+        return path, contents
+
+    @staticmethod
+    def _get_output_path(url) -> Path:
+        if is_url(url):
+            return Path(slugify(url))
+
+        return Path(url)
+
+    def _get_from_cache(self, caching, url):
+        if caching == CachingEnum.NEVER:
+            return None
+
+        cached_value = self.cache_manager.get(url)
+        if cached_value is not None:
+            logger.debug(f"Using cached value for URL {url}")
+            return cached_value
+
+        return None
+
+    def _do_fetch(self, url):
+        raise NotImplementedError()
+
+    def _save_to_cache(self, caching, caching_delta, url, contents):
+        if caching == CachingEnum.FOREVER:
+            logger.debug(f"Caching forever the contents of {url}")
+            self.cache_manager.forever(str(url), contents)
+        elif caching == CachingEnum.EXPIRES:
+            future = datetime.now() + caching_delta
+            logger.debug(f"Caching the contents of {url} to expire in {future}")
+            self.cache_manager.put(str(url), contents, future)
+
+
+FetchersType = Dict[str, "StyleFetcher"]
diff --git a/src/nitpick/style/fetchers/file.py b/src/nitpick/style/fetchers/file.py
new file mode 100644
index 00000000..bb164cd8
--- /dev/null
+++ b/src/nitpick/style/fetchers/file.py
@@ -0,0 +1,17 @@
+"""Basic local file fetcher."""
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Tuple
+
+from nitpick.style.fetchers.base import StyleFetcher
+
+
+@dataclass(repr=True, unsafe_hash=True)
+class FileFetcher(StyleFetcher):  # pylint: disable=too-few-public-methods
+    """Fetch a style from a local file."""
+
+    protocols: Tuple[str, ...] = ("file", "")
+
+    def _do_fetch(self, url):
+        file_path = Path(url).expanduser()
+        return file_path.read_text()
diff --git a/src/nitpick/style/fetchers/github.py b/src/nitpick/style/fetchers/github.py
new file mode 100644
index 00000000..b3a4abb8
--- /dev/null
+++ b/src/nitpick/style/fetchers/github.py
@@ -0,0 +1,47 @@
+"""Support for ``gh`` and ``github`` schemes."""
+from dataclasses import dataclass
+from typing import Tuple
+from urllib.parse import urlparse, uses_netloc, uses_relative
+
+from nitpick.style.fetchers.http import HttpFetcher
+
+
+@dataclass(repr=True, unsafe_hash=True)
+class GitHubFetcher(HttpFetcher):  # pylint: disable=too-few-public-methods
+    """Fetch styles from GitHub repositories."""
+
+    _api_url = "https://api.github.com/repos/{owner}/{repository}"
+    _download_url = "https://raw.githubusercontent.com/{owner}/{repository}/{git_reference}{path}"
+    _registered = False
+    protocols: Tuple[str, ...] = ("gh", "github")
+
+    def _post_hooks(self):
+        self._register_on_urllib()
+
+    @classmethod
+    def _register_on_urllib(cls):
+        if cls._registered:
+            return
+
+        # This is necessary so urljoin knows how to deal with custom protocols
+        for protocol in cls.protocols:
+            uses_relative.append(protocol)
+            uses_netloc.append(protocol)
+
+    def _download(self, url):
+        parsed_url = urlparse(url)
+        owner = parsed_url.netloc
+        repository = parsed_url.path.split("/")[1]
+        main_branch = self._get_main_branch(owner, repository)
+
+        download_url = self._download_url.format(
+            owner=owner,
+            repository=repository,
+            git_reference=main_branch,
+            path=parsed_url.path.replace(f"/{repository}", ""),
+        )
+        return super()._download(download_url)
+
+    def _get_main_branch(self, owner, repository):
+        response = self._session.get(self._api_url.format(owner=owner, repository=repository)).json()
+        return response["default_branch"]
diff --git a/src/nitpick/style/fetchers/http.py b/src/nitpick/style/fetchers/http.py
new file mode 100644
index 00000000..c970f09a
--- /dev/null
+++ b/src/nitpick/style/fetchers/http.py
@@ -0,0 +1,49 @@
+"""Base HTTP fetcher, other fetchers can inherit from this to wrap http errors."""
+from dataclasses import dataclass, field
+from typing import Tuple
+
+import click
+import requests
+from loguru import logger
+from requests.sessions import Session
+
+from nitpick.enums import OptionEnum
+from nitpick.style.fetchers import _requests_session
+from nitpick.style.fetchers.base import StyleFetcher
+
+
+@dataclass(repr=True, unsafe_hash=True)
+class HttpFetcher(StyleFetcher):
+    """Fetch a style from an http/https server."""
+
+    _session: Session = field(init=False)
+
+    requires_connection = True
+    protocols: Tuple[str, ...] = ("http", "https")
+
+    def __post_init__(self):
+        """Sessions should be per class as children can have custom headers or authentication."""
+        self._session = _requests_session()
+        self._post_hooks()
+
+    def _post_hooks(self):
+        """Dataclasses won't call post init here if another class extends it and override."""
+
+    def _do_fetch(self, url) -> str:
+        try:
+            contents = self._download(url)
+        except requests.ConnectionError as err:
+            logger.exception(f"Request failed with {err}")
+            click.secho(
+                "Your network is unreachable. Fix your connection or use"
+                f" {OptionEnum.OFFLINE.as_flake8_flag()} / {OptionEnum.OFFLINE.as_envvar()}=1",
+                fg="red",
+                err=True,
+            )
+            return ""
+        return contents
+
+    def _download(self, url) -> str:
+        response = self._session.get(url)
+        response.raise_for_status()
+        return response.text


Test patch:
diff --git a/tests/test_style.py b/tests/test_style.py
index 3912f669..3dc39973 100644
--- a/tests/test_style.py
+++ b/tests/test_style.py
@@ -659,3 +659,49 @@ def test_invalid_nitpick_files(offline, tmp_path):
         """,
         2,
     )
+
+
+@responses.activate
+def test_github_fetch(tmp_path):
+    """Test that gh:// and github:// URLs can be fetched."""
+    raw_url = "https://raw.githubusercontent.com/andreoliwa/nitpick/develop"
+    data = [
+        (
+            f"{raw_url}/initial.toml",
+            """
+            [nitpick.styles]
+            include = "black.toml"
+            """,
+        ),
+        (
+            f"{raw_url}/black.toml",
+            """
+            ["pyproject.toml".tool.black]
+            line-length = 120
+            """,
+        ),
+    ]
+    for url, style in data:
+        responses.add(responses.GET, url, dedent(style), status=200)
+
+    responses.add(responses.GET, "https://api.github.com/repos/andreoliwa/nitpick", """{"default_branch": "develop"}""")
+
+    ProjectMock(tmp_path).pyproject_toml(
+        """
+        [tool.nitpick]
+        style = [
+          "github://andreoliwa/nitpick/initial.toml",
+        ]
+        """
+    ).api_check().assert_violations(
+        Fuss(
+            False,
+            PYPROJECT_TOML,
+            318,
+            " has missing values:",
+            """
+            [tool.black]
+            line-length = 120
+            """,
+        )
+    )