Repository: mondeja/mkdocs-include-markdown-plugin
Pull request #165 (2023-07-31 21:49:25)

Add `cache` extra for persistent HTTP requests file caching


Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d5899b5..771b01a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -76,7 +76,8 @@ jobs:
       - name: Install dependencies
         run: pip install -U hatch
       - name: Run tests
-        run: hatch run +py=${{ matrix.py }} tests:all
+        run: |
+          hatch run +py=${{ matrix.py }} tests:all
       - name: Convert coverage to XML
         run: |
           pip install coverage covdefaults
diff --git a/examples/http-cache/docs/README.md b/examples/http-cache/docs/README.md
new file mode 100644
index 0000000..69ff55d
--- /dev/null
+++ b/examples/http-cache/docs/README.md
@@ -0,0 +1,15 @@
+# Header
+
+You need to install `platformdirs` to run this example.
+
+{%
+  include-markdown "https://raw.githubusercontent.com/mondeja/mkdocs-include-markdown-plugin/master/examples/basic/docs/included.md"
+%}
+
+{%
+  include-markdown "https://raw.githubusercontent.com/mondeja/mkdocs-include-markdown-plugin/master/examples/basic/mkdocs.yml"
+%}
+
+{%
+  include-markdown "https://raw.githubusercontent.com/mondeja/mkdocs-include-markdown-plugin/master/examples/basic/docs/included.md"
+%}
diff --git a/examples/http-cache/mkdocs.yml b/examples/http-cache/mkdocs.yml
new file mode 100644
index 0000000..9a6cd80
--- /dev/null
+++ b/examples/http-cache/mkdocs.yml
@@ -0,0 +1,4 @@
+site_name: Foo
+plugins:
+  - include-markdown:
+      cache: 600
diff --git a/pyproject.toml b/pyproject.toml
index 5284811..b9992ac 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -44,6 +44,9 @@ Changelog = "https://github.com/mondeja/mkdocs-include-markdown-plugin/releases"
 [project.entry-points."mkdocs.plugins"]
 include-markdown = "mkdocs_include_markdown_plugin.plugin:IncludeMarkdownPlugin"
 
+[project.optional-dependencies]
+cache = ["platformdirs"]
+
 [tool.hatch.build]
 include = ["/src"]
 
@@ -69,6 +72,7 @@ dependencies = ["pytest~=7.0", "coverage~=6.4", "covdefaults"]
 [[tool.hatch.envs.tests.matrix]]
 python = ["py37", "py38", "py39", "py310", "py311", "py312"]
 mkdocs = ["1.2.4", "1.3.1", "1.4.0", "1.4.2"]
+cache = ["yes", "no"]
 
 [tool.hatch.envs.tests.overrides]
 matrix.mkdocs.dependencies = [
@@ -77,6 +81,9 @@ matrix.mkdocs.dependencies = [
   { value = "mkdocs==1.4.0", if = ["1.4.0"] },
   { value = "mkdocs==1.4.2", if = ["1.4.2"] },
 ]
+matrix.cache.dependencies = [
+  { value = "platformdirs", if = ["yes"] },
+]
 
 [tool.hatch.envs.tests.scripts]
 all = "coverage run -m pytest"
diff --git a/src/mkdocs_include_markdown_plugin/cache.py b/src/mkdocs_include_markdown_plugin/cache.py
new file mode 100644
index 0000000..4daa9b2
--- /dev/null
+++ b/src/mkdocs_include_markdown_plugin/cache.py
@@ -0,0 +1,83 @@
+"""Persistent file cache."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import os
+import time
+
+
+class Cache:
+    """Cache for arbitrary content, one file per entry."""
+
+    def __init__(self, cache_dir: str, expiration_seconds: int = 0):
+        self.cache_dir = cache_dir
+        self.expiration_seconds = expiration_seconds
+
+    def get_creation_time_from_fpath(self, fpath: str) -> int:
+        """Get creation time of an entry in the cache given its path."""
+        with open(fpath, encoding='utf-8') as f:
+            return int(f.readline().strip())
+
+    @classmethod
+    def generate_unique_key_from_url(cls, url: str) -> str:
+        """Generate a unique key from an URL."""
+        return base64.urlsafe_b64encode(
+            hashlib.sha3_512(url.encode()).digest(),
+        ).decode('utf-8')
+
+    def read_file(self, fpath: str) -> str:  # noqa: D102
+        with open(fpath, encoding='utf-8') as f:
+            return '\n'.join(f.read().split('\n')[1:])
+
+    def get_(self, url: str) -> str | None:  # noqa: D102
+        key = self.generate_unique_key_from_url(url)
+        fpath = os.path.join(self.cache_dir, key)
+        if os.path.isfile(fpath):
+            creation_time = self.get_creation_time_from_fpath(fpath)
+            if time.time() > creation_time + self.expiration_seconds:
+                return self.read_file(fpath)
+            else:
+                os.remove(fpath)
+                return None
+        else:
+            return None
+
+    def set_(self, url: str, value: str) -> None:  # noqa: D102
+        key = self.generate_unique_key_from_url(url)
+        fpath = os.path.join(self.cache_dir, key)
+        with open(fpath, 'w', encoding='utf-8') as f:
+            f.write(f'{int(time.time())}\n')
+            f.write(value)
+
+    def clean(self) -> None:
+        """Clean expired entries from the cache."""
+        for fname in os.listdir(self.cache_dir):
+            fpath = os.path.join(self.cache_dir, fname)
+            creation_time = self.get_creation_time_from_fpath(fpath)
+            if time.time() > creation_time + self.expiration_seconds:
+                os.remove(fpath)
+
+
+def get_cache_directory() -> str | None:
+    """Get the cache directory."""
+    try:
+        from platformdirs import user_data_dir
+    except ImportError:
+        return None
+
+    cache_dir = user_data_dir('mkdocs-include-markdown-plugin')
+    if not os.path.isdir(cache_dir):
+        os.makedirs(cache_dir)
+
+    return cache_dir
+
+
+def initialize_cache(expiration_seconds: int) -> Cache | None:
+    """Initialize a cache instance."""
+    cache_dir = get_cache_directory()
+    if cache_dir is None:
+        return None
+
+    return Cache(cache_dir, expiration_seconds)
diff --git a/src/mkdocs_include_markdown_plugin/config.py b/src/mkdocs_include_markdown_plugin/config.py
index 8e93e94..ff3a59a 100644
--- a/src/mkdocs_include_markdown_plugin/config.py
+++ b/src/mkdocs_include_markdown_plugin/config.py
@@ -21,6 +21,7 @@ CONFIG_DEFAULTS = {
     'dedent': False,
     'trailing_newlines': True,
     'comments': DEFAULT_COMMENTS,
+    'cache': 0,
 }
 
 CONFIG_SCHEME = (
@@ -52,6 +53,10 @@ CONFIG_SCHEME = (
         'comments',
         MkType(bool, default=DEFAULT_COMMENTS),
     ),
+    (
+        'cache',
+        MkType(int, default=CONFIG_DEFAULTS['cache']),
+    ),
 )
 
 
diff --git a/src/mkdocs_include_markdown_plugin/event.py b/src/mkdocs_include_markdown_plugin/event.py
index eb32de1..adc6478 100644
--- a/src/mkdocs_include_markdown_plugin/event.py
+++ b/src/mkdocs_include_markdown_plugin/event.py
@@ -16,6 +16,7 @@ from urllib.parse import urlparse
 from mkdocs.exceptions import BuildError
 
 from mkdocs_include_markdown_plugin import process
+from mkdocs_include_markdown_plugin.cache import Cache
 from mkdocs_include_markdown_plugin.config import (
     CONFIG_DEFAULTS,
     DEFAULT_CLOSING_TAG,
@@ -137,11 +138,18 @@ def read_file(file_path: str, encoding: str) -> str:
         return f.read()
 
 
-def read_http(target_url: str, encoding: str) -> Any:  # noqa: U100
+def read_http(target_url: str, http_cache: Cache | None) -> Any:  # noqa: U100
     """Read an http location and return its content."""
+    if http_cache is not None:
+        cached_content = http_cache.get_(target_url)
+        if cached_content is not None:
+            return cached_content
     req = urllib.request.Request(target_url)
     with urllib.request.urlopen(req) as response:
-        return response.read().decode('UTF-8')
+        content = response.read().decode('UTF-8')
+    if http_cache is not None:
+        http_cache.set_(target_url, content)
+    return content
 
 
 def get_file_content(
@@ -157,6 +165,7 @@ def get_file_content(
     default_comments: bool = DEFAULT_COMMENTS,
     cumulative_heading_offset: int = 0,
     files_watcher: FilesWatcher | None = None,
+    http_cache: Cache | None = None,
 ) -> str:
     """Return the content of the file to include."""
     def found_include_tag(match: re.Match[str]) -> str:
@@ -235,20 +244,6 @@ def get_file_content(
         elif files_watcher is not None:
             if not is_url(file_path_glob):
                 files_watcher.included_files.extend(file_paths_to_include)
-            else:
-                readable_files_to_include = ', '.join(file_paths_to_include)
-                plural_suffix = 's' if len(file_paths_to_include) > 1 else ''
-                lineno = lineno_from_content_start(
-                    markdown,
-                    directive_match_start,
-                )
-                logger.warning(
-                    f'Not adding a watcher for {file_path_glob} of'
-                    " 'include-markdown' directive at"
-                    f' {os.path.relpath(page_src_path, docs_dir)}'
-                    f':{lineno} not detected in the file{plural_suffix}'
-                    f' {readable_files_to_include}',
-                )
 
         bool_options: dict[str, DirectiveBoolArgument] = {
             'preserve-includer-indent': {
@@ -336,7 +331,7 @@ def get_file_content(
         expected_but_any_found = [start is not None, end is not None]
         for file_path in file_paths_to_include:
             if is_url(filename):
-                new_text_to_include = read_http(file_path, encoding)
+                new_text_to_include = read_http(file_path, http_cache)
             else:
                 new_text_to_include = read_file(file_path, encoding)
 
@@ -364,6 +359,7 @@ def get_file_content(
                 default_dedent,
                 default_trailing_newlines,
                 files_watcher=files_watcher,
+                http_cache=http_cache,
             )
 
             # trailing newlines right stripping
@@ -489,20 +485,6 @@ def get_file_content(
         elif files_watcher is not None:
             if not is_url(file_path_glob):
                 files_watcher.included_files.extend(file_paths_to_include)
-            else:
-                readable_files_to_include = ', '.join(file_paths_to_include)
-                plural_suffix = 's' if len(file_paths_to_include) > 1 else ''
-                lineno = lineno_from_content_start(
-                    markdown,
-                    directive_match_start,
-                )
-                logger.warning(
-                    f'Not adding a watcher for {file_path_glob}'
-                    " of 'include-markdown' directive at"
-                    f' {os.path.relpath(page_src_path, docs_dir)}'
-                    f':{lineno} not detected in the file{plural_suffix}'
-                    f' {readable_files_to_include}',
-                )
 
         bool_options: dict[str, DirectiveBoolArgument] = {
             'rewrite-relative-urls': {
@@ -624,7 +606,7 @@ def get_file_content(
         text_to_include = ''
         for file_path in file_paths_to_include:
             if is_url(filename):
-                new_text_to_include = read_http(file_path, encoding)
+                new_text_to_include = read_http(file_path, http_cache)
             else:
                 new_text_to_include = read_file(file_path, encoding)
 
@@ -653,6 +635,7 @@ def get_file_content(
                 default_trailing_newlines,
                 default_comments=default_comments,
                 files_watcher=files_watcher,
+                http_cache=http_cache,
             )
 
             # trailing newlines right stripping
@@ -743,6 +726,7 @@ def on_page_markdown(
     docs_dir: str,
     config: MutableMapping[str, Any] | None = None,
     files_watcher: FilesWatcher | None = None,
+    http_cache: Cache | None = None,
 ) -> str:
     """Process markdown content of a page."""
     if config is None:
@@ -776,4 +760,5 @@ def on_page_markdown(
         config.get('trailing_newlines', CONFIG_DEFAULTS['trailing_newlines']),
         default_comments=config.get('comments', CONFIG_DEFAULTS['comments']),
         files_watcher=files_watcher,
+        http_cache=config.get('_cache', http_cache),
     )
diff --git a/src/mkdocs_include_markdown_plugin/plugin.py b/src/mkdocs_include_markdown_plugin/plugin.py
index 6afb53a..1ea6896 100644
--- a/src/mkdocs_include_markdown_plugin/plugin.py
+++ b/src/mkdocs_include_markdown_plugin/plugin.py
@@ -5,6 +5,7 @@ from __future__ import annotations
 from collections.abc import Callable
 from typing import TYPE_CHECKING, Any, TypeVar, cast
 
+from mkdocs.exceptions import BuildError
 from mkdocs.livereload import LiveReloadServer
 from mkdocs.plugins import BasePlugin
 
@@ -22,6 +23,7 @@ if TYPE_CHECKING:
     from mkdocs.structure.files import Files
     from mkdocs.structure.pages import Page
 
+from mkdocs_include_markdown_plugin.cache import initialize_cache
 from mkdocs_include_markdown_plugin.config import (
     CONFIG_SCHEME,
     create_include_tag,
@@ -51,6 +53,19 @@ class IncludeMarkdownPlugin(BasePlugin):
             tag='include-markdown',
         )
 
+        if self.config['cache'] > 0:
+            cache = initialize_cache(self.config['cache'])
+            if cache is None:
+                raise BuildError(
+                    'The "platformdirs" package is required to use the'
+                    ' "cache" option. Install'
+                    ' mkdocs-include-markdown-plugin with the "cache"'
+                    ' extra to install it.',
+                )
+            else:
+                cache.clean()
+                self.config['_cache'] = cache
+
         return config
 
     def _watch_included_files(self) -> None:


Test patch:
diff --git a/tests/test_integration/test_cache.py b/tests/test_integration/test_cache.py
new file mode 100644
index 0000000..7c1448e
--- /dev/null
+++ b/tests/test_integration/test_cache.py
@@ -0,0 +1,73 @@
+import importlib
+import os
+
+import pytest
+from testing_helpers import parametrize_directives
+
+from mkdocs_include_markdown_plugin.cache import (
+    Cache,
+    get_cache_directory,
+    initialize_cache,
+)
+from mkdocs_include_markdown_plugin.event import on_page_markdown
+
+
+@pytest.mark.parametrize(
+    ('url', 'expected_result'),
+    (
+        (
+            'https://raw.githubusercontent.com/mondeja/mkdocs-include-markdown-plugin/master/examples/basic/mkdocs.yml',  # noqa: E501
+            '''site_name: Foo
+plugins:
+  - include-markdown
+''',
+        ),
+        (
+            'https://raw.githubusercontent.com/mondeja/mkdocs-include-markdown-plugin/master/examples/basic/docs/included.md',  # noqa: E501
+            '''Some ignored content.
+
+<--start-->
+
+Some included content.
+''',
+        ),
+    ),
+)
+@parametrize_directives
+def test_page_included_by_url_is_cached(
+    directive,
+    url,
+    expected_result,
+    page,
+    tmp_path,
+):
+    cache_dir = get_cache_directory()
+    try:
+        importlib.import_module('platformdirs')
+    except ImportError:
+        assert cache_dir is None
+        assert initialize_cache(600) is None
+        return
+
+    file_path = os.path.join(
+        cache_dir, Cache.generate_unique_key_from_url(url),
+    )
+    if os.path.isfile(file_path):
+        os.remove(file_path)
+
+    cache = Cache(cache_dir, 600)
+
+    result = on_page_markdown(
+        f'''{{%
+    {directive} "{url}"
+    comments=false
+%}}''',
+        page(tmp_path / 'includer.md'),
+        tmp_path,
+        http_cache=cache,
+    )
+
+    assert result == expected_result
+
+    assert os.path.isfile(file_path)
+    os.remove(file_path)
diff --git a/tests/test_integration/test_examples.py b/tests/test_integration/test_examples.py
index 429b392..4b0b05e 100644
--- a/tests/test_integration/test_examples.py
+++ b/tests/test_integration/test_examples.py
@@ -1,3 +1,4 @@
+import importlib
 import os
 import subprocess
 import sys
@@ -11,14 +12,25 @@ EXAMPLES_DIR = os.path.join(rootdir, 'examples')
 
 @pytest.mark.parametrize('dirname', os.listdir(EXAMPLES_DIR))
 def test_examples(dirname):
+    expected_returncode = 0
+
+    example_dir = os.path.join(EXAMPLES_DIR, dirname)
+    config_file = os.path.join(example_dir, 'mkdocs.yml')
+    with open(config_file, encoding='utf-8') as f:
+        if 'cache:' in f.read():
+            try:
+                importlib.import_module('platformdirs')
+            except ImportError:
+                expected_returncode = 1
+
     proc = subprocess.Popen(
         [sys.executable, '-mmkdocs', 'build'],
-        cwd=os.path.join(EXAMPLES_DIR, dirname),
+        cwd=example_dir,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
     stdout, stderr = proc.communicate()
 
-    assert proc.returncode == 0, (
+    assert proc.returncode == expected_returncode, (
         f'{stdout.decode("utf-8")}\n{stderr.decode("utf-8")}'
     )