Repository: aeye-lab/pymovements
Pull request #325 (2023-04-15 20:24:49)

Dataset: Use curly braces format instead of pure regex
## Description

I added curly braces format for specifying dataset files.

I minimally changed specified functionality by adding need to specify type of value (by default it's string), I think this change is logical because strings are also default option in `filename_format_dtypes`.

So now instead of:
```
r'(?P<subject_id>\d+)_(?P<session_id>\d+).csv'
```
it can be written as:
```
r'{subject_id:d}_{session_id:d}.csv'
```
All other nuances are specified in function's docstring.

Also, old regex method should still work if needed, this change only adds functionality.

Fixes issue #319

## Implemented changes

- Implemented function `curly_to_regex(s: str) -> re.Pattern` in utils.strings;
- Added unit tests for `curly_to_regex` function (100% coverage);
- Replaced `DatasetDefinition.filename_regex` with `DatasetDefinition.filename_format`;
- Also replaced `DatasetDefinition.filename_regex_dtypes` with `DatasetDefinition.filename_format_dtypes`;
- Changed one docs tutorial accordingly, however I'm not shure if further documentation changes are not needed.

## Type of change

- [x] New feature (non-breaking change which adds functionality)
- [x] Breaking change (fix or feature that would cause existing functionality to not work as expected)
- [x] This change is or requires a documentation update

## How Has This Been Tested?

I added unit tests and ran `tox` command, all tests passed.

## Checklist:

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my feature works
- [x] New and existing unit tests pass locally with my changes
- [x] I have checked my code and corrected any misspellings



Patch:
diff --git a/docs/source/tutorials/local-dataset.ipynb b/docs/source/tutorials/local-dataset.ipynb
index 34438a9..bbd4494 100644
--- a/docs/source/tutorials/local-dataset.ipynb
+++ b/docs/source/tutorials/local-dataset.ipynb
@@ -114,7 +114,7 @@
    "id": "cd6ab2fe",
    "metadata": {},
    "source": [
-    "We also define a `filename_regex` which is a regular expression used to match and extract values from filenames of data files in the dataset. For example, `r'trial_(?P<text_id>\\d+)_(?P<page_id>\\d+).csv'` will match filenames that follow the pattern `trial_{text_id}_{page_id}.csv` and extract the values of `text_id` and `page_id` for each file."
+    "We also define a `filename_format` which is a pattern expression used to match and extract values from filenames of data files in the dataset. For example, `r'trial_{text_id:d}_{page_id:d}.csv'` will match filenames that follow the pattern `trial_{text_id}_{page_id}.csv` and extract the values of `text_id` and `page_id` for each file."
    ]
   },
   {
@@ -124,7 +124,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "filename_regex = r'trial_(?P<text_id>\\d+)_(?P<page_id>\\d+).csv'"
+    "filename_format = r'trial_{text_id:d}_{page_id:d}.csv'"
    ]
   },
   {
@@ -142,7 +142,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "filename_regex_dtypes = {\n",
+    "filename_format_dtypes = {\n",
     "    'text_id': int,\n",
     "    'page_id': int,\n",
     "}"
@@ -224,8 +224,8 @@
     "dataset_definition = pm.DatasetDefinition(\n",
     "    name='my_dataset',\n",
     "    experiment=experiment,\n",
-    "    filename_regex=filename_regex,\n",
-    "    filename_regex_dtypes=filename_regex_dtypes,\n",
+    "    filename_format=filename_format,\n",
+    "    filename_format_dtypes=filename_format_dtypes,\n",
     "    column_map=column_map,\n",
     "    custom_read_kwargs=custom_read_kwargs,\n",
     ")"
diff --git a/src/pymovements/dataset/dataset_definition.py b/src/pymovements/dataset/dataset_definition.py
index 4bb510f..5719093 100644
--- a/src/pymovements/dataset/dataset_definition.py
+++ b/src/pymovements/dataset/dataset_definition.py
@@ -48,13 +48,13 @@ class DatasetDefinition:
     experiment : Experiment
         The experiment definition.
 
-    filename_regex : str
+    filename_format : str
         Regular expression which will be matched before trying to load the file. Namedgroups will
         appear in the `fileinfo` dataframe.
 
-    filename_regex_dtypes : dict[str, type], optional
-        If named groups are present in the `filename_regex`, this makes it possible to cast specific
-        named groups to a particular datatype.
+    filename_format_dtypes : dict[str, type], optional
+        If named groups are present in the `filename_format`, this makes it possible to cast
+        specific named groups to a particular datatype.
 
     column_map : dict[str, str]
         The keys are the columns to read, the values are the names to which they should be renamed.
@@ -70,9 +70,9 @@ class DatasetDefinition:
 
     experiment: Experiment | None = None
 
-    filename_regex: str = '.*'
+    filename_format: str = '.*'
 
-    filename_regex_dtypes: dict[str, type] = field(default_factory=dict)
+    filename_format_dtypes: dict[str, type] = field(default_factory=dict)
 
     custom_read_kwargs: dict[str, Any] = field(default_factory=dict)
 
diff --git a/src/pymovements/dataset/dataset_files.py b/src/pymovements/dataset/dataset_files.py
index 09980ea..f47e942 100644
--- a/src/pymovements/dataset/dataset_files.py
+++ b/src/pymovements/dataset/dataset_files.py
@@ -20,7 +20,6 @@
 """Functionality to scan, load and save dataset files."""
 from __future__ import annotations
 
-import re
 from pathlib import Path
 from typing import Any
 
@@ -32,6 +31,7 @@ from pymovements.dataset.dataset_paths import DatasetPaths
 from pymovements.events.events import EventDataFrame
 from pymovements.gaze.gaze_dataframe import GazeDataFrame
 from pymovements.utils.paths import match_filepaths
+from pymovements.utils.strings import curly_to_regex
 
 
 def scan_dataset(definition: DatasetDefinition, paths: DatasetPaths) -> pl.DataFrame:
@@ -59,7 +59,7 @@ def scan_dataset(definition: DatasetDefinition, paths: DatasetPaths) -> pl.DataF
     # Get all filepaths that match regular expression.
     fileinfo_dicts = match_filepaths(
         path=paths.raw,
-        regex=re.compile(definition.filename_regex),
+        regex=curly_to_regex(definition.filename_format),
         relative=True,
     )
 
@@ -72,7 +72,7 @@ def scan_dataset(definition: DatasetDefinition, paths: DatasetPaths) -> pl.DataF
 
     fileinfo_df = fileinfo_df.with_columns([
         pl.col(fileinfo_key).cast(fileinfo_dtype)
-        for fileinfo_key, fileinfo_dtype in definition.filename_regex_dtypes.items()
+        for fileinfo_key, fileinfo_dtype in definition.filename_format_dtypes.items()
     ])
 
     return fileinfo_df
@@ -259,7 +259,7 @@ def add_fileinfo(
     # Cast columns from fileinfo according to specification.
     df = df.with_columns([
         pl.col(fileinfo_key).cast(fileinfo_dtype)
-        for fileinfo_key, fileinfo_dtype in definition.filename_regex_dtypes.items()
+        for fileinfo_key, fileinfo_dtype in definition.filename_format_dtypes.items()
     ])
     return df
 
diff --git a/src/pymovements/datasets/gazebase.py b/src/pymovements/datasets/gazebase.py
index 19ec095..92e5839 100644
--- a/src/pymovements/datasets/gazebase.py
+++ b/src/pymovements/datasets/gazebase.py
@@ -65,13 +65,13 @@ class GazeBase(DatasetDefinition):
     experiment : Experiment
         The experiment definition.
 
-    filename_regex : str
+    filename_format : str
         Regular expression which will be matched before trying to load the file. Namedgroups will
         appear in the `fileinfo` dataframe.
 
-    filename_regex_dtypes : dict[str, type], optional
-        If named groups are present in the `filename_regex`, this makes it possible to cast specific
-        named groups to a particular datatype.
+    filename_format_dtypes : dict[str, type], optional
+        If named groups are present in the `filename_format`, this makes it possible to cast
+        specific named groups to a particular datatype.
 
     column_map : dict[str, str]
         The keys are the columns to read, the values are the names to which they should be renamed.
@@ -123,13 +123,13 @@ class GazeBase(DatasetDefinition):
         sampling_rate=1000,
     )
 
-    filename_regex: str = (
-        r'S_(?P<round_id>\d)(?P<subject_id>\d+)'
-        r'_S(?P<session_id>\d+)'
-        r'_(?P<task_name>.+).csv'
+    filename_format: str = (
+        r'S_{round_id:1d}{subject_id:d}'
+        r'_S{session_id:d}'
+        r'_{task_name}.csv'
     )
 
-    filename_regex_dtypes: dict[str, type] = field(
+    filename_format_dtypes: dict[str, type] = field(
         default_factory=lambda: {
             'round_id': int,
             'subject_id': int,
diff --git a/src/pymovements/datasets/judo1000.py b/src/pymovements/datasets/judo1000.py
index 7d42838..0f002b9 100644
--- a/src/pymovements/datasets/judo1000.py
+++ b/src/pymovements/datasets/judo1000.py
@@ -58,13 +58,13 @@ class JuDo1000(DatasetDefinition):
     experiment : Experiment
         The experiment definition.
 
-    filename_regex : str
+    filename_format : str
         Regular expression which will be matched before trying to load the file. Namedgroups will
         appear in the `fileinfo` dataframe.
 
-    filename_regex_dtypes : dict[str, type], optional
-        If named groups are present in the `filename_regex`, this makes it possible to cast specific
-        named groups to a particular datatype.
+    filename_format_dtypes : dict[str, type], optional
+        If named groups are present in the `filename_format`, this makes it possible to cast
+        specific named groups to a particular datatype.
 
     column_map : dict[str, str]
         The keys are the columns to read, the values are the names to which they should be renamed.
@@ -116,9 +116,9 @@ class JuDo1000(DatasetDefinition):
         sampling_rate=1000,
     )
 
-    filename_regex: str = r'(?P<subject_id>\d+)_(?P<session_id>\d+).csv'
+    filename_format: str = r'{subject_id:d}_{session_id:d}.csv'
 
-    filename_regex_dtypes: dict[str, type] = field(
+    filename_format_dtypes: dict[str, type] = field(
         default_factory=lambda: {
             'subject_id': int,
             'session_id': int,
diff --git a/src/pymovements/datasets/toy_dataset.py b/src/pymovements/datasets/toy_dataset.py
index efbc9b7..4a57bb4 100644
--- a/src/pymovements/datasets/toy_dataset.py
+++ b/src/pymovements/datasets/toy_dataset.py
@@ -56,13 +56,13 @@ class ToyDataset(DatasetDefinition):
     experiment : Experiment
         The experiment definition.
 
-    filename_regex : str
+    filename_format : str
         Regular expression which will be matched before trying to load the file. Namedgroups will
         appear in the `fileinfo` dataframe.
 
-    filename_regex_dtypes : dict[str, type], optional
-        If named groups are present in the `filename_regex`, this makes it possible to cast specific
-        named groups to a particular datatype.
+    filename_format_dtypes : dict[str, type], optional
+        If named groups are present in the `filename_format`, this makes it possible to cast
+        specific named groups to a particular datatype.
 
     column_map : dict[str, str]
         The keys are the columns to read, the values are the names to which they should be renamed.
@@ -114,9 +114,9 @@ class ToyDataset(DatasetDefinition):
         sampling_rate=1000,
     )
 
-    filename_regex: str = r'trial_(?P<text_id>\d+)_(?P<page_id>\d+).csv'
+    filename_format: str = r'trial_{text_id:d}_{page_id:d}.csv'
 
-    filename_regex_dtypes: dict[str, type] = field(
+    filename_format_dtypes: dict[str, type] = field(
         default_factory=lambda: {
             'text_id': int,
             'page_id': int,
diff --git a/src/pymovements/utils/strings.py b/src/pymovements/utils/strings.py
new file mode 100644
index 0000000..799ba5e
--- /dev/null
+++ b/src/pymovements/utils/strings.py
@@ -0,0 +1,61 @@
+# Copyright (c) 2022-2023 The pymovements Project Authors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""
+This module holds string specific funtions.
+"""
+from __future__ import annotations
+
+import re
+
+CURLY_TO_REGEX = re.compile(
+    r'(?:^|(?<=[^{])|(?<={{)){(?P<name>[^{][0-9a-zA-z_]*?)(?:\:(?P<quantity>\d*)(?P<type>[sd])?)?}',
+)
+
+
+def curly_to_regex(s: str) -> re.Pattern:
+    """
+    Returns regex pattern converted from provided python formatting style pattern.
+    By default all parameters are strings, if you want to specify number you can do: {num:d}
+    If you want to specify parameter's length you can do: {two_symbols:2} or {four_digits:4d}
+    Characters { and } can be escaped the same way as in python: {{ and }}
+    For example:
+                r'{subject_id:d}_{session_name}.csv'
+    converts to r'(?P<subject_id>[0-9]+)_(?P<session_name>.+).csv'
+
+    Parameters
+    ----------
+    s: str
+        Pattern in python formatting style.
+    """
+
+    def replace_aux(match: re.Match) -> str:     # Auxiliary replacement function
+        pattern = r'.'
+        if match.group('type') == 'd':
+            pattern = r'[0-9]'
+        elif match.group('type') == 's':
+            pattern = r'.'
+        quantity = r'+'
+        if match.group('quantity'):
+            quantity = f'{{{match.group("quantity")}}}'
+        return fr'(?P<{match.group("name")}>{pattern}{quantity})'
+
+    result = CURLY_TO_REGEX.sub(replace_aux, s)
+    result = result.replace('{{', '{').replace('}}', '}')
+    return re.compile(result)


Test patch:
diff --git a/tests/dataset/dataset_download_test.py b/tests/dataset/dataset_download_test.py
index c94a5f5..3e97395 100644
--- a/tests/dataset/dataset_download_test.py
+++ b/tests/dataset/dataset_download_test.py
@@ -309,6 +309,6 @@ def test_public_dataset_registered_correct_attributes(tmp_path, dataset_definiti
     assert dataset.definition.mirrors == dataset_definition.mirrors
     assert dataset.definition.resources == dataset_definition.resources
     assert dataset.definition.experiment == dataset_definition.experiment
-    assert dataset.definition.filename_regex == dataset_definition.filename_regex
-    assert dataset.definition.filename_regex_dtypes == dataset_definition.filename_regex_dtypes
+    assert dataset.definition.filename_format == dataset_definition.filename_format
+    assert dataset.definition.filename_format_dtypes == dataset_definition.filename_format_dtypes
     assert dataset.definition.custom_read_kwargs == dataset_definition.custom_read_kwargs
diff --git a/tests/dataset/dataset_test.py b/tests/dataset/dataset_test.py
index 14b2af3..65ba06e 100644
--- a/tests/dataset/dataset_test.py
+++ b/tests/dataset/dataset_test.py
@@ -230,8 +230,8 @@ def mock_toy(rootpath, raw_fileformat, eyes):
             origin='lower left',
             sampling_rate=1000,
         ),
-        filename_regex=r'(?P<subject_id>\d+).' + raw_fileformat,
-        filename_regex_dtypes={'subject_id': pl.Int64},
+        filename_format=r'{subject_id:d}.' + raw_fileformat,
+        filename_format_dtypes={'subject_id': pl.Int64},
     )
 
     return {
diff --git a/tests/datasets/datasets_test.py b/tests/datasets/datasets_test.py
index 073d652..dab9fd3 100644
--- a/tests/datasets/datasets_test.py
+++ b/tests/datasets/datasets_test.py
@@ -55,6 +55,6 @@ def test_public_dataset_registered_correct_attributes(dataset_definition_class):
     assert dataset_definition.mirrors == registered_definition.mirrors
     assert dataset_definition.resources == registered_definition.resources
     assert dataset_definition.experiment == registered_definition.experiment
-    assert dataset_definition.filename_regex == registered_definition.filename_regex
-    assert dataset_definition.filename_regex_dtypes == registered_definition.filename_regex_dtypes
+    assert dataset_definition.filename_format == registered_definition.filename_format
+    assert dataset_definition.filename_format_dtypes == registered_definition.filename_format_dtypes
     assert dataset_definition.custom_read_kwargs == registered_definition.custom_read_kwargs
diff --git a/tests/utils/strings_test.py b/tests/utils/strings_test.py
new file mode 100644
index 0000000..39c3bae
--- /dev/null
+++ b/tests/utils/strings_test.py
@@ -0,0 +1,98 @@
+# Copyright (c) 2023 The pymovements Project Authors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""
+Test pymovements string utilities.
+"""
+from __future__ import annotations
+
+import re
+
+import pytest
+
+from pymovements.utils.strings import curly_to_regex
+
+
+@pytest.mark.parametrize(
+    'pattern, expected_regex',
+    [
+        pytest.param(
+            r'{subject_id:d}_{session_name}.csv',
+            re.compile(r'(?P<subject_id>[0-9]+)_(?P<session_name>.+).csv'),
+            id='test_basic_pattern',
+        ),
+        pytest.param(
+            r'',
+            re.compile(r''),
+            id='test_empty_string',
+        ),
+        pytest.param(
+            r'test',
+            re.compile(r'test'),
+            id='test_no_curly_braces',
+        ),
+        pytest.param(
+            r'{test}',
+            re.compile(r'(?P<test>.+)'),
+            id='test_one_curly_brace',
+        ),
+        pytest.param(
+            r'{t3ST}',
+            re.compile(r'(?P<t3ST>.+)'),
+            id='test_various_characters_as_name',
+        ),
+        pytest.param(
+            r'{test1}_{test2}',
+            re.compile(r'(?P<test1>.+)_(?P<test2>.+)'),
+            id='test_two_curly_braces',
+        ),
+        pytest.param(
+            r'{{{test1}}}',
+            re.compile(r'{(?P<test1>.+)}'),
+            id='test_nested_curly_braces',
+        ),
+        pytest.param(
+            r'{{test1}}',
+            re.compile(r'{test1}'),
+            id='test_escaped_curly_braces',
+        ),
+        pytest.param(
+            r'{num:s}',
+            re.compile(r'(?P<num>.+)'),
+            id='test_explicit_string',
+        ),
+        pytest.param(
+            r'{num:d}',
+            re.compile(r'(?P<num>[0-9]+)'),
+            id='test_numbers',
+        ),
+        pytest.param(
+            r'{num:42}',
+            re.compile(r'(?P<num>.{42})'),
+            id='test_quantities',
+        ),
+        pytest.param(
+            r'{num:5d}',
+            re.compile(r'(?P<num>[0-9]{5})'),
+            id='test_quantities_2',
+        ),
+    ],
+)
+def test_curly_to_regex(pattern, expected_regex):
+    assert curly_to_regex(pattern) == expected_regex