Repository: ImperialCollegeLondon/pycsvy

Issue:
Implement validators registry
## Problem

The yaml header is meant to be flexible and there is no imposition of any specific fields that must be there. However, there are fields that, if present, must follow certain schema. An example of this are the CSV dialect (that will be implemented in #2) and the Tabular schema (implemented in #3). It is easy to see that other applications and tools might require their own strict set of fields, eg. an measurement equipment that provides as metadata machine make and model, start and end time of the measurements, and some specific machine settings. All of those must be there if the metadata is to truly represent the data it is accompanying. 

## Proposed solution

Implement a validator registry that will run specific, known validators on the header if a given field is present. Eg. assuming #3 is implemented

```python
@register_validator(name="tabular")
class TabularSchema(...)
    ...

# and then, when reading the yaml header
for field, validator in VALIDATORS_REGISTRY.items():
    if field in yaml_header:
        yaml_header[field] = validator(field)
```
So the TabularSchema validator will run if `tabular` is a top level field in the yaml header. The same would be done when saving the data, but in that case, the validator would be dumping the data to a dict, so it can be saved.

This issue is not about implementing any specific validator, but about setting up the infrastructure for running them. 

Discussion:

Patch:
diff --git a/.vscode/settings.json b/.vscode/settings.json
index e137fad..e36fe4b 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,4 +1,21 @@
 {
-    "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true
-}
\ No newline at end of file
+  "[python]": {
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  },
+  "[json]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[jsonc]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[yaml]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[toml]": {
+    "editor.formatOnSave": false
+  },
+  "editor.formatOnSave": true,
+  "editor.rulers": [88],
+  "python.testing.unittestEnabled": false,
+  "python.testing.pytestEnabled": true
+}
diff --git a/csvy/readers.py b/csvy/readers.py
index 7cec3f5..ae34cbb 100644
--- a/csvy/readers.py
+++ b/csvy/readers.py
@@ -34,6 +34,8 @@ except ModuleNotFoundError:
         "Polars is not installed. Reading into a pl.DataFrame will not work."
     )
 
+from .validators import validate_read
+
 
 def get_comment(line: str, marker: str = "---") -> str:
     """Retrieves the comment character used in the header.
@@ -90,7 +92,7 @@ def read_header(
             line = line.lstrip(comment)
             header.append(line)
 
-    return yaml.safe_load("".join(header), **kwargs), nlines, comment
+    return validate_read(yaml.safe_load("".join(header), **kwargs)), nlines, comment
 
 
 def read_metadata(
diff --git a/csvy/validators.py b/csvy/validators.py
index 103f18c..7911550 100644
--- a/csvy/validators.py
+++ b/csvy/validators.py
@@ -1,14 +1,91 @@
 """This module contains validators for the CSVY file format."""
 
 import csv
-from typing import Optional, TypeVar
+from typing import Any, Callable, Optional, TypeVar
 
 from pydantic import BaseModel, Field
 
+VALIDATORS_REGISTRY: dict[str, type[BaseModel]] = {}
+"""Registry of validators to run on the header."""
+
+
+def register_validator(
+    name: str, overwrite: bool = False
+) -> Callable[[type[BaseModel]], type[BaseModel]]:
+    """Registers a validator in the registry.
+
+    This function is a decorator that registers a validator in the registry. The name
+    of the validator is used as the key in the registry.
+
+    Args:
+        name: The name of the validator.
+        overwrite: Whether to overwrite the validator if it already exists.
+
+    Returns:
+        The decorator function that registers the validator.
+    """
+
+    def decorator(cls: type[BaseModel]) -> type[BaseModel]:
+        if not issubclass(cls, BaseModel):
+            raise TypeError("Validators must be subclasses of pydantic.BaseModel.")
+
+        if name in VALIDATORS_REGISTRY and not overwrite:
+            raise ValueError(f"Validator with name '{name}' already exists.")
+
+        VALIDATORS_REGISTRY[name] = cls
+        return cls
+
+    return decorator
+
+
+def validate_read(header: dict[str, Any]) -> dict[str, Any]:
+    """Runs the validators on the header in a read operation.
+
+    This function runs the validators on the header. It uses the keys of the header to
+    find the validators in the registry and runs them on the corresponding values.
+
+    Args:
+        header: The header of the CSVY file.
+
+    Returns:
+        The validated header.
+    """
+    validated_header = {}
+    for key, value in header.items():
+        if key in VALIDATORS_REGISTRY:
+            validator = VALIDATORS_REGISTRY[key]
+            validated_header[key] = validator(**value)
+        else:
+            validated_header[key] = value
+    return validated_header
+
+
+def validate_write(header: dict[str, Any]) -> dict[str, Any]:
+    """Uses the validators to create the header in a write operation.
+
+    Transforms the header with validators to a header with dictionaries that can be
+    saved as yaml. It is the reversed operation of validate_read, so calling
+    validate_write(validate_read(header)) should return the original header.
+
+    Args:
+        header: Dictionary to be saved as the header of the CSVY file.
+
+    Returns:
+        The validated header.
+    """
+    validated_header = {}
+    for key, value in header.items():
+        validated_header[key] = (
+            value.model_dump() if isinstance(value, BaseModel) else value
+        )
+    return validated_header
+
+
 # Create a generic variable that can be 'Parent', or any subclass.
 T = TypeVar("T", bound="CSVDialectValidator")
 
 
+@register_validator("csv_dialect")
 class CSVDialectValidator(BaseModel):
     r"""Implements a validator for CSV Dialects.
 
diff --git a/csvy/writers.py b/csvy/writers.py
index 068971d..a5a3d17 100644
--- a/csvy/writers.py
+++ b/csvy/writers.py
@@ -11,6 +11,8 @@ from typing import Any
 
 import yaml
 
+from .validators import validate_write
+
 KNOWN_WRITERS: list[Callable[[Path | str, Any, str], bool]] = []
 
 
@@ -133,15 +135,16 @@ def write_header(
         **kwargs: Arguments to pass to 'yaml.safe_dump'. If "sort_keys" is not one of
             arguments, it will be set to sort_keys=False.
     """
+    header_ = validate_write(header)
     if not isinstance(file, TextIOBase):
         with Path(file).open("w") as f:
-            write_header(f, header, comment, **kwargs)
+            write_header(f, header_, comment, **kwargs)
             return
 
     if "sort_keys" not in kwargs:
         kwargs["sort_keys"] = False
 
-    stream = yaml.safe_dump(header, **kwargs)
+    stream = yaml.safe_dump(header_, **kwargs)
     stream = "\n".join([f"{comment}" + line for line in stream.split("\n")])
     marker = f"{comment}---\n"
     stream = marker + stream + "---\n"
diff --git a/setup.cfg b/setup.cfg
index 52c4052..3337cd1 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -26,6 +26,7 @@ extend-ignore =
 [mypy]
 ignore_missing_imports = True
 strict_optional = False
+check_untyped_defs = True
 
 [mypy-setup]
 ignore_errors = True


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 8562954..439441b 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -25,3 +25,14 @@ def array_data_path():
     from pathlib import Path
 
     return Path(__file__).parent / "array_data.csv"
+
+
+@pytest.fixture
+def validators_registry():
+    """A validators registry fixture."""
+    from csvy.validators import VALIDATORS_REGISTRY
+
+    backup = VALIDATORS_REGISTRY.copy()
+    yield VALIDATORS_REGISTRY
+    VALIDATORS_REGISTRY.clear()
+    VALIDATORS_REGISTRY.update(backup)
diff --git a/tests/test_read.py b/tests/test_read.py
index 9d2041f..f0768bb 100644
--- a/tests/test_read.py
+++ b/tests/test_read.py
@@ -109,7 +109,7 @@ def test_read_to_polars(data_path):
 
     import csvy.readers as readers
 
-    readers.LazyFrame = None
+    readers.LazyFrame = None  # type: ignore [misc]
 
     with pytest.raises(ModuleNotFoundError):
         read_to_polars(data_path)
diff --git a/tests/test_validators.py b/tests/test_validators.py
index a71d58c..073eeee 100644
--- a/tests/test_validators.py
+++ b/tests/test_validators.py
@@ -20,3 +20,107 @@ def test_shortcut_dialects_roundtrip(shortcut):
     assert dialect.lineterminator == actual.lineterminator
     assert dialect.quotechar == actual.quotechar
     assert dialect.skipinitialspace == actual.skipinitialspace
+
+
+def test_register_validator(validators_registry):
+    """Test that we can register a new validator."""
+    from pydantic import BaseModel
+
+    from csvy.validators import register_validator
+
+    @register_validator("my_validator")
+    class MyValidator(BaseModel):
+        pass
+
+    assert validators_registry["my_validator"] == MyValidator
+
+
+def test_register_validator_duplicate(validators_registry):
+    """Test that we cannot register a validator with the same name."""
+    from pydantic import BaseModel
+
+    from csvy.validators import register_validator
+
+    # With overwriting, we should not raise an error.
+    name = "my_validator"
+
+    @register_validator(name)
+    class _(BaseModel):
+        pass
+
+    # Without overwriting, we should raise an error.
+    with pytest.raises(ValueError):
+
+        @register_validator(name)
+        class _(BaseModel):  # type: ignore [no-redef]
+            pass
+
+    # With overwriting, we should not raise an error,
+    # and the validator should be overwritten.
+    @register_validator(name, overwrite=True)
+    class MyNewOverwritingValidator(BaseModel):
+        pass
+
+    assert validators_registry[name] == MyNewOverwritingValidator
+
+
+def test_register_validator_not_base_model(validators_registry):
+    """Test that we cannot register a validator that is not a BaseModel."""
+    from csvy.validators import register_validator
+
+    with pytest.raises(TypeError):
+
+        @register_validator("not_base_model")  # type: ignore [arg-type]
+        class _:
+            pass
+
+
+def test_validate_read(validators_registry):
+    """Test that we can run validators on the header."""
+    from pydantic import BaseModel, PositiveInt
+
+    from csvy.validators import register_validator, validate_read
+
+    @register_validator("my_validator")
+    class MyValidator(BaseModel):
+        value: PositiveInt
+
+    header = {"author": "Gandalf", "my_validator": {"value": 42}}
+    validated_header = validate_read(header)
+
+    assert isinstance(validated_header["my_validator"], MyValidator)
+    assert validated_header["my_validator"].value == 42
+    assert validated_header["author"] == header["author"]
+
+
+def test_validate_read_missing(validators_registry):
+    """Test that we can run validators on the header."""
+    from pydantic import BaseModel, PositiveInt, ValidationError
+
+    from csvy.validators import register_validator, validate_read
+
+    @register_validator("my_validator")
+    class _(BaseModel):
+        value: PositiveInt
+
+    header = {"author": "Gandalf", "my_validator": {}}
+
+    with pytest.raises(ValidationError):
+        validate_read(header)
+
+
+def test_validate_write(validators_registry):
+    """Test that we can create the header using the validators."""
+    from pydantic import BaseModel, PositiveInt
+
+    from csvy.validators import register_validator, validate_read, validate_write
+
+    @register_validator("my_validator")
+    class _(BaseModel):
+        value: PositiveInt
+
+    header = {"author": "Gandalf", "my_validator": {"value": 42}}
+    validated_header = validate_read(header)
+    new_header = validate_write(validated_header)
+
+    assert new_header == header
diff --git a/tests/test_write.py b/tests/test_write.py
index 08962bf..ecb1c28 100644
--- a/tests/test_write.py
+++ b/tests/test_write.py
@@ -59,11 +59,11 @@ def test_write_numpy(mock_save, tmpdir):
 
     filename = tmpdir / "some_file.csv"
 
-    data = []
+    data: list = []
     assert not write_numpy(filename, data)
 
-    data = np.array([])
-    assert write_numpy(filename, data)
+    data2 = np.array([])
+    assert write_numpy(filename, data2)
     mock_save.assert_called_once()
 
 
@@ -76,7 +76,7 @@ def test_write_pandas(mock_save, tmpdir):
 
     filename = tmpdir / "some_file.csv"
 
-    data = []
+    data: list = []
     assert not write_pandas(filename, data)
 
     data = pd.DataFrame()
@@ -93,17 +93,17 @@ def test_write_polars(mock_save, tmpdir, mocker):
 
     filename = tmpdir / "some_file.csv"
 
-    data = []
+    data: list = []
     assert not write_polars(filename, data)
 
-    data = pl.DataFrame()
-    assert write_polars(filename, data)
+    data2 = pl.DataFrame()
+    assert write_polars(filename, data2)
     mock_save.assert_called_once()
 
-    data = pl.LazyFrame()
-    collect_spy = mocker.spy(data, "collect")
+    data3 = pl.LazyFrame()
+    collect_spy = mocker.spy(data3, "collect")
     mock_save.reset_mock()
-    assert write_polars(filename, data)
+    assert write_polars(filename, data3)
     collect_spy.assert_called_once()
     mock_save.assert_called_once()
 
@@ -167,7 +167,7 @@ def test_writer_writerow(mock_write_header, mock_csv_writer, tmpdir):
 
     data = (1, 2, 3)
     writer.writerow(data)
-    writer._writer.writerow.assert_called_once_with(data)
+    writer._writer.writerow.assert_called_once_with(data)  # type: ignore [attr-defined]
 
 
 @patch("csv.writer")
@@ -181,7 +181,7 @@ def test_writer_writerows(mock_write_header, mock_csv_writer, tmpdir):
 
     data = ((1, 2, 3),)
     writer.writerows(data)
-    writer._writer.writerows.assert_called_once_with(data)
+    writer._writer.writerows.assert_called_once_with(data)  # type: ignore [attr-defined]
 
 
 def test_writer_close(tmpdir):
@@ -250,11 +250,15 @@ def test_write_data(mock_write_csv):
     KNOWN_WRITERS.clear()
     KNOWN_WRITERS.append(MagicMock(return_value=True))
     write_data(filename, data, comment, **csv_options)
-    KNOWN_WRITERS[0].assert_called_once_with(filename, data, comment, **csv_options)
+    KNOWN_WRITERS[0].assert_called_once_with(  # type: ignore [attr-defined]
+        filename, data, comment, **csv_options
+    )
     mock_write_csv.assert_not_called()
 
     KNOWN_WRITERS.clear()
     KNOWN_WRITERS.append(MagicMock(return_value=False))
     write_data(filename, data, comment, **csv_options)
-    KNOWN_WRITERS[0].assert_called_once_with(filename, data, comment, **csv_options)
+    KNOWN_WRITERS[0].assert_called_once_with(  # type: ignore [attr-defined]
+        filename, data, comment, **csv_options
+    )
     mock_write_csv.assert_called_once_with(filename, data, comment, **csv_options)