Repository: encukou/freezeyt (language: python)

Issue:
Přidat možnost vybrat databázi/knihovnu na mimetypes
Funkce `freeze` by měla brát nový argument, `guess_type`, což bude funkce, která se chová jako `mimetypes.guess_type` a použije se místo ní.


Discussion / hints:
Misto argumentu tyhle veci ted mame v konfiguraci.

Fix patch:
diff --git a/README.md b/README.md
index 0428ce07..e07f03bd 100644
--- a/README.md
+++ b/README.md
@@ -267,13 +267,15 @@ extra_files:
 Extra files cannot be specified on the CLI.
 
 
-### Default MIME type
+### Comparison of MIME type and file type
 
 Freezeyt checks whether the file extensions in its output
 correspond to the MIME types served by the app.
 If there's a mismatch, freezeyt fails, because this means a server
 wouldn't be able to serve the page correctly.
 
+#### Default MIME type
+
 It is possible to specify the MIME type used for files without an extension.
 For example, if your server of static pages defaults to plain text files,
 use:
@@ -282,6 +284,39 @@ use:
 default_mimetype=text/plain
 ```
 
+If the default MIME type isn't explicitly configured in YAML configuration,
+then the `freezeyt` uses value `application/octet-stream`.
+
+#### Recognizing file types from extensions
+
+There is possibility to modify the way how to determine file type
+from file extension.
+You can setup your own `get_mimetype` function.
+
+Freezeyt will register your own function, if you specify it in configuration
+YAML file as:
+
+```yaml
+get_mimetype=module:your_function
+```
+
+If the `get_mimetype` is not defined in configuration file,
+then `freezeyt` calls the python function `mimetypes.guess_type`
+and uses the mimetype (the first element) it returns.
+
+`get_mimetype` can be defined as:
+* strings in the form `"module:function"`, which name the function to call,
+* Python functions (if configuring `freezeyt` from Python, e.g. as a `dict`,
+  rather than YAML).
+
+The `get_mimetype`:
+*gets one argument the `filepath` as `string`
+
+*returns file type as `string` (e.g. `"text/plain"`).
+
+If `get_mimetypes` returns `None`, `freezeyt` will use the configured `default_mimetype` (see *Default MIME type* above).
+
+
 ### Progress bar and logging
 
 The CLI option `--progress` controls what `freezeyt` outputs as it
diff --git a/freezeyt/freezer.py b/freezeyt/freezer.py
index 23fcc695..49d037e3 100644
--- a/freezeyt/freezer.py
+++ b/freezeyt/freezer.py
@@ -59,11 +59,22 @@ async def freeze_async(app, config):
 }
 
 
-def check_mimetype(url_path, headers, default='application/octet-stream'):
+
+def default_get_mimetype(url: str) -> Optional[str]:
+    """Returns filetype as a string from mimetype.guess_type
+    """
+    file_type, encoding = guess_type(url)
+    return file_type
+
+
+def check_mimetype(
+    url_path, headers,
+    default='application/octet-stream', *, get_mimetype=default_get_mimetype,
+):
     if url_path.endswith('/'):
         # Directories get saved as index.html
         url_path = 'index.html'
-    file_type, file_encoding = guess_type(url_path)
+    file_type = get_mimetype(url_path)
     if not file_type:
         file_type = default
     headers = Headers(headers)
@@ -177,8 +188,21 @@ def __init__(self, app, config):
 
         self.freeze_info = hooks.FreezeInfo(self)
 
-        self.extra_pages = config.get('extra_pages', ())
-        self.extra_files = config.get('extra_files', None)
+        CONFIG_DATA = (
+            ('extra_pages', ()),
+            ('extra_files', None),
+            ('default_mimetype', 'application/octet-stream'),
+            ('get_mimetype', default_get_mimetype),
+            ('url_to_path', default_url_to_path)
+        )
+        for attr_name, default in CONFIG_DATA:
+            setattr(self, attr_name, config.get(attr_name, default))
+
+        if isinstance(self.get_mimetype, str):
+            self.get_mimetype = import_variable_from_module(self.get_mimetype)
+
+        if isinstance(self.url_to_path, str):
+            self.url_to_path = import_variable_from_module(self.url_to_path)
 
         if config.get('use_default_url_finders', True):
             _url_finders = dict(
@@ -223,10 +247,6 @@ def __init__(self, app, config):
         else:
             raise ValueError(f"unknown output type {output['type']}")
 
-        self.url_to_path = config.get('url_to_path', default_url_to_path)
-        if isinstance(self.url_to_path, str):
-            self.url_to_path = import_variable_from_module(self.url_to_path)
-
         # The tasks for individual pages are tracked in the followng sets
         # (actually dictionaries: {task.path: task})
         # Each task must be in exactly in one of these.
@@ -396,9 +416,8 @@ def start_response(
         if status_action == 'save':
             check_mimetype(
                 url.path, headers,
-                default=self.config.get(
-                    'default_mimetype', 'application/octet-stream',
-                ),
+                default=self.default_mimetype,
+                get_mimetype=self.get_mimetype
             )
             return wsgi_write
         elif status_action == 'ignore':


Test patch:
diff --git a/tests/test_check_mimetype.py b/tests/test_check_mimetype.py
index 66ba6870..10d02494 100644
--- a/tests/test_check_mimetype.py
+++ b/tests/test_check_mimetype.py
@@ -2,157 +2,193 @@
 
 from freezeyt.freezer import check_mimetype
 
-def test_normal_html():
-    check_mimetype(
+
+TEST_DATA = {
+    'normal_html': (
         'http://localhost:8000/index.html',
         [
             ('Content-Type', 'text/html; charset=utf-8'),
             ('Content-Length', '164'),
-        ]
-    )
-
-def test_diff_type_jpg_png_fail():
-    with pytest.raises(ValueError):
-        check_mimetype(
-            'http://localhost:8000/image.jpg',
-            [
-                ('Content-Type', 'image/png'),
-                ('Content-Length', '654'),
-            ],
-        )
-
-
-def test_missing_content_type():
-    with pytest.raises(ValueError):
-        check_mimetype(
-            'http://localhost:8000/index.html',
-            [
-                ('Content-Length', '164'),
-            ]
-        )
-
+        ],
+    ),
 
-def test_case_insensitive_content_type():
-        check_mimetype(
-            'http://localhost:8000/index.html',
-            [
-                ('Content-Type', 'TEXT/HTML; charset=utf-8'),
-                ('Content-Length', '164'),
-            ]
-        )
+    'jpg_X_png_fail': (
+        'http://localhost:8000/index.jpg',
+        [
+            ('Content-Type', 'image/png'),
+            ('Content-Length', '654'),
+        ],
+    ),
 
+    'missing_content_type_fail': (
+        'http://localhost:8000/index.html', [('Content-Length', '164')]
+    ),
 
-def test_case_insensitive_file():
-        check_mimetype(
-            'http://localhost:8000/index.HTML',
-            [
-                ('Content-Type', 'text/html; charset=utf-8'),
-                ('Content-Length', '164'),
-            ]
-        )
+    'case_insensitive_content_type': (
+        'http://localhost:8000/index.html',
+        [
+            ('Content-Type', 'TEXT/HTML; charset=utf-8'),
+            ('Content-Length', '164'),
+        ],
+    ),
 
+    'case_insensitive_file': (
+        'http://localhost:8000/index.HTML',
+        [
+            ('Content-Type', 'text/html; charset=utf-8'),
+            ('Content-Length', '164'),
+        ],
+    ),
 
-def test_case_insensitive_headers_fail():
-    with pytest.raises(ValueError):
-        check_mimetype(
-            'http://localhost:8000/image.jpg',
-            [
-                ('CONTENT-TYPE', 'IMAGE/PNG'),
-                ('Content-Length', '654'),
-            ],
-        )
+    'case_insensitive_headers_fail': (
+        'http://localhost:8000/index.jpg',
+        [
+            ('CONTENT-TYPE', 'IMAGE/PNG'),
+            ('Content-Length', '654'),
+        ],
+    ),
 
+    'case_insensitive_headers': (
+        'http://localhost:8000/index.jpg',
+        [
+            ('coNtENT-TypE', 'IMAgE/JpEG'),
+            ('Content-Length', '654'),
+        ],
+    ),
 
-def test_case_insensitive_headers():
-    check_mimetype(
+    # The content-type for JPEG images is standardized: `image/jpeg` (case insensitive).
+    # The file extension is not standardized; the default recognizer guesses that
+    # both `.jpg` or `.jpeg` mean JPEG images.
+    'same_jpg_fail': (
         'http://localhost:8000/image.jpg',
         [
-            ('coNtENT-TypE', 'IMAgE/JpEG'),
+            ('Content-Type', 'image/jpg'),
             ('Content-Length', '654'),
         ],
-    )
+    ),
 
+    'missing_file_suffix': (
+        'http://localhost:8000/index',
+        [
+            ('Content-Type', 'application/octet-stream'),
+            ('Content-Length', '164'),
+        ],
+    ),
 
-def test_same_jpg_fail():
-    with pytest.raises(ValueError):
-        check_mimetype(
-            'http://localhost:8000/image.jpg',
-            [
-                ('Content-Type', 'image/jpg'),
-                ('Content-Length', '654'),
-            ],
-        )
+    'missing_file_suffix_fail': (
+        'http://localhost:8000/index',
+        [
+            ('Content-Type', 'image/png'),
+            ('Content-Length', '164'),
+        ],
+    ),
 
+    'directory': (
+        'http://localhost:8000/foo/', [('Content-Type', 'text/html')]
+    ),
+}
 
-def test_missing_file_suffix():
-    check_mimetype(
+@pytest.mark.parametrize('testname', TEST_DATA)
+def test_kwargs_exclude(testname):
+    url_path, headers= TEST_DATA[testname]
+    error = testname[-5:] == '_fail'
+
+    if error:
+        with pytest.raises(ValueError):
+            check_mimetype(url_path, headers)
+    else:
+        check_mimetype(url_path, headers)
+
+
+TEST_DATA_DEFAULT = {
+    'missing_file_suffix': (
         'http://localhost:8000/index',
         [
-            ('Content-Type', 'application/octet-stream'),
+            ('Content-Type', 'image/png'),
             ('Content-Length', '164'),
-        ]
-    )
-
-def test_missing_file_suffix_fail():
-    with pytest.raises(ValueError):
-        check_mimetype(
-            'http://localhost:8000/index',
-            [
-                ('Content-Type', 'image/png'),
-                ('Content-Length', '164'),
-            ]
-        )
+        ],
+        'image/png'
+    ),
 
-def test_directory():
-    check_mimetype(
-        'http://localhost:8000/foo/',
+    'missing_file_suffix_fail': (
+        'http://localhost:8000/index',
         [
             ('Content-Type', 'text/html'),
-        ]
-    )
+            ('Content-Length', '164'),
+        ],
+        'image/png'
+    ),
 
+    'default_ignored': (
+        'http://localhost:8000/picture.jpg',
+        [
+            ('Content-Type', 'image/jpeg'),
+            ('Content-Length', '164'),
+        ],
+        'image/png'
+    ),
 
-def test_missing_file_default():
-    check_mimetype(
-        'http://localhost:8000/index',
+    'default_ignored_fail': (
+        'http://localhost:8000/picture.jpg',
         [
             ('Content-Type', 'image/png'),
             ('Content-Length', '164'),
         ],
-        default='image/png'
+        'image/png'
     )
+}
+
+@pytest.mark.parametrize('testname', TEST_DATA_DEFAULT)
+def test_default_include(testname):
+    url_path, headers, default = TEST_DATA_DEFAULT[testname]
+    error = testname[-5:] == '_fail'
+
+    if error:
+        with pytest.raises(ValueError):
+            check_mimetype(url_path, headers, default=default)
+    else:
+        check_mimetype(url_path, headers, default=default)
 
 
-def test_default_ignored():
+def test_missing_file_suffix_get_mimetype():
+    def get_mimetype(url_path):
+        return "image/png"
+
     check_mimetype(
-        'http://localhost:8000/picture.jpg',
+        'http://localhost:8000/index',
         [
-            ('Content-Type', 'image/jpeg'),
+            ('Content-Type', 'image/png'),
             ('Content-Length', '164'),
         ],
-        default='image/png'
+        get_mimetype=get_mimetype
     )
 
+def test_missing_file_suffix_get_mimetype_fail():
+    def get_mimetype(url_path):
+        return "image/png"
 
-def test_missing_file_default_fail():
     with pytest.raises(ValueError):
         check_mimetype(
             'http://localhost:8000/index',
             [
-                ('Content-Type', 'text/html'),
+                ('Content-Type', 'image/jpeg'),
                 ('Content-Length', '164'),
             ],
-            default='image/png'
+            get_mimetype=get_mimetype
         )
 
+def test_missing_file_suffix_get_mimetype_fail_default_ignored():
+    def get_mimetype(url_path):
+        return "image/bmp"
 
-def test_default_ignored_fail():
     with pytest.raises(ValueError):
         check_mimetype(
-            'http://localhost:8000/picture.jpg',
+            'http://localhost:8000/index',
             [
-                ('Content-Type', 'image/png'),
+                ('Content-Type', 'text/html'),
                 ('Content-Length', '164'),
             ],
-            default='image/png'
+            default='text/html',
+            get_mimetype=get_mimetype,
+
         )
diff --git a/tests/test_get_mimetype.py b/tests/test_get_mimetype.py
new file mode 100644
index 00000000..cad2b130
--- /dev/null
+++ b/tests/test_get_mimetype.py
@@ -0,0 +1,35 @@
+import pytest
+from pathlib import Path
+from freezeyt import freeze
+from testutil import context_for_test
+
+
+def my_get_mimetype(url_path):
+    suffix = Path(url_path).suffix
+
+    if suffix == '.html':
+        return "text/html"
+    return "text/plain"
+
+GET_MIMETYPES = {
+    'python_get_mimetype': my_get_mimetype,
+    'string_get_mimetype': 'test_get_mimetype:my_get_mimetype',
+}
+
+
+@pytest.mark.parametrize('get_mimetypeID', GET_MIMETYPES)
+def test_succesfully_loaded_get_mimetype_config(tmp_path, get_mimetypeID):
+    """Test
+    """
+    builddir = tmp_path / 'build'
+
+    with context_for_test('default_mimetype_plain') as module:
+        freeze_config = {
+            'output': str(builddir),
+            'get_mimetype': GET_MIMETYPES[get_mimetypeID],
+        }
+
+        freeze(module.app, freeze_config)
+
+        assert (builddir / 'index.html').exists()
+        assert (builddir / 'textfile').exists()