Repository: aeye-lab/pymovements
Pull request #636 (2023-12-04 11:17:00)

feat: add PoTeC dataset
## Description

This PR adds the PoTeC dataset to pymovements.

## Checklist:

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my feature works
- [x] New and existing unit tests pass locally with my changes
- [x] Any dependent changes have been merged and published in downstream modules
- [x] I have checked my code and corrected any misspellings


prassepaul: > Somehow the dataset ist not part of `DatasetLibrary` and I don't get why.
> 
> * [x]  Please add the respective test cases to this file:
>   https://github.com/aeye-lab/pymovements/blob/main/tests/unit/datasets/datasets_test.py
> * [x]  make sure that the integration test passes
> 
> You can run a single test case like this:
> 
> ```shell
> pytest tests/integration/public_dataset_processing_test.py::test_public_dataset_processing[PoTeC]
> ```

To perform the pytest, you need to install pymovements from path....

Patch:
diff --git a/docs/source/bibliography.bib b/docs/source/bibliography.bib
index 9b43a31..8ba08da 100644
--- a/docs/source/bibliography.bib
+++ b/docs/source/bibliography.bib
@@ -35,6 +35,14 @@
     doi = {10.17605/OSF.IO/5ZPVK},
 }
 
+@misc{potec,
+    author = {Jäger, Lena A. and Haller, Patrick and Jakobi, Deborah Noemie},
+    title = {{Potsdam Textbook Corpus}},
+    year = {2021},
+    pages = {},
+    doi = {10.17605/OSF.IO/DN5HP},
+}
+
 @article{SalvucciGoldberg2000,
     author = {Salvucci, Dario D. and Goldberg, Joseph H.},
     title = {Identifying Fixations and Saccades in Eye-Tracking Protocols},
diff --git a/src/pymovements/dataset/dataset_files.py b/src/pymovements/dataset/dataset_files.py
index eea25d4..758c639 100644
--- a/src/pymovements/dataset/dataset_files.py
+++ b/src/pymovements/dataset/dataset_files.py
@@ -101,9 +101,10 @@ def load_event_files(
     events_dirname: str | None
         One-time usage of an alternative directory name to save data relative to dataset path.
         This argument is used only for this single call and does not alter
-        :py:meth:`pymovements.Dataset.events_rootpath`. (default: None)
-    extension: str
-        Specifies the file format for loading data. Valid options are: `csv`, `feather`.
+        :py:meth:`pymovements.Dataset.events_rootpath`.
+    extension:
+        Specifies the file format for loading data. Valid options are: `csv`, `feather`,
+        `tsv`, `txt`.
         (default: 'feather')
 
     Returns
@@ -132,10 +133,10 @@ def load_event_files(
 
         if extension == 'feather':
             event_df = pl.read_ipc(filepath)
-        elif extension == 'csv':
+        elif extension in {'csv', 'tsv', 'txt'}:
             event_df = pl.read_csv(filepath)
         else:
-            valid_extensions = ['csv', 'feather']
+            valid_extensions = ['csv', 'txt', 'tsv', 'feather']
             raise ValueError(
                 f'unsupported file format "{extension}".'
                 f'Supported formats are: {valid_extensions}',
@@ -178,10 +179,11 @@ def load_gaze_files(
         One-time usage of an alternative directory name to save data relative to
         :py:meth:`pymovements.Dataset.path`.
         This argument is used only for this single call and does not alter
-        :py:meth:`pymovements.Dataset.preprocessed_rootpath`. (default: None)
-    extension: str
-        Specifies the file format for loading data. Valid options are: `csv`, `feather`.
-        :Default: `feather`. (default: 'feather')
+        :py:meth:`pymovements.Dataset.preprocessed_rootpath`.
+    extension:
+        Specifies the file format for loading data. Valid options are: `csv`, `feather`,
+        `txt`, `tsv`.
+        (default: 'feather')
 
     Returns
     -------
@@ -232,7 +234,7 @@ def load_gaze_files(
                 trial_columns=definition.trial_columns,
             )
 
-        elif preprocessed and extension == 'csv':
+        elif preprocessed and extension in {'csv', 'txt', 'tsv'}:
             time_column = None
             distance_column = None
 
@@ -317,7 +319,7 @@ def load_gaze_file(
     if custom_read_kwargs is None:
         custom_read_kwargs = {}
 
-    if filepath.suffix in {'.csv', '.txt'}:
+    if filepath.suffix in {'.csv', '.txt', '.tsv'}:
         if preprocessed:
             gaze_df = pl.read_csv(filepath)
         else:
@@ -327,7 +329,7 @@ def load_gaze_file(
     elif filepath.suffix == '.asc':
         gaze_df = parse_eyelink(filepath, **custom_read_kwargs)
     else:
-        valid_extensions = ['csv', 'txt', 'feather', 'asc']
+        valid_extensions = ['csv', 'tsv', 'txt', 'feather', 'asc']
         raise ValueError(
             f'unsupported file format "{filepath.suffix}".'
             f'Supported formats are: {valid_extensions}',
@@ -403,7 +405,7 @@ def save_events(
         (default: 1)
     extension: str
         Specifies the file format for loading data. Valid options are: `csv`, `feather`.
-        :Default: `feather`. (default: 'feather')
+        (default: 'feather')
 
     Raises
     ------
diff --git a/src/pymovements/datasets/__init__.py b/src/pymovements/datasets/__init__.py
index 0bc602c..9139d26 100644
--- a/src/pymovements/datasets/__init__.py
+++ b/src/pymovements/datasets/__init__.py
@@ -31,6 +31,7 @@
     pymovements.datasets.GazeOnFaces
     pymovements.datasets.HBN
     pymovements.datasets.JuDo1000
+    pymovements.datasets.PoTeC
     pymovements.datasets.SBSAT
 
 
@@ -49,6 +50,7 @@ from pymovements.datasets.gazebasevr import GazeBaseVR
 from pymovements.datasets.gaze_graph import GazeGraph
 from pymovements.datasets.hbn import HBN
 from pymovements.datasets.judo1000 import JuDo1000
+from pymovements.datasets.potec import PoTeC
 from pymovements.datasets.sb_sat import SBSAT
 from pymovements.datasets.toy_dataset import ToyDataset
 from pymovements.datasets.toy_dataset_eyelink import ToyDatasetEyeLink
@@ -61,6 +63,7 @@ __all__ = [
     'GazeOnFaces',
     'HBN',
     'JuDo1000',
+    'PoTeC',
     'SBSAT',
     'ToyDataset',
     'ToyDatasetEyeLink',
diff --git a/src/pymovements/datasets/potec.py b/src/pymovements/datasets/potec.py
new file mode 100644
index 0000000..f682f07
--- /dev/null
+++ b/src/pymovements/datasets/potec.py
@@ -0,0 +1,160 @@
+# Copyright (c) 2022-2023 The pymovements Project Authors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""This module provides an interface to the PoTeC dataset."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Any
+
+import polars as pl
+
+from pymovements.dataset.dataset_definition import DatasetDefinition
+from pymovements.dataset.dataset_library import register_dataset
+from pymovements.gaze.experiment import Experiment
+
+
+@dataclass
+@register_dataset
+class PoTeC(DatasetDefinition):
+    """PoTeC dataset :cite:p:`potec`.
+
+    The Potsdam Textbook Corpus (PoTeC) is a corpus of eye-tracking-while-reading data where
+    participants (N=75) read a series of German short texts taken from college level textbooks
+    of physics and biology. The experiments were conducted within a 2x2 fully-crossed factorial
+    design with the reader’s expertise (advanced vs beginner) and major (physics vs biology) as
+    factors. Reading comprehension was assessed using text comprehension questions. Moreover,
+    background questions that required additional knowledge beyond the presented text tested the
+    general domain knowledge.
+    The repository contains the eye-movement data (1000 Hz, right eye monocular) as well as the
+    stimulus text data     with extensive linguistic feature annotations at the sub-lexical,
+    lexical und supra-lexical     level. Therefore, the PoTeC is ideal for studying cognitive
+    processes related to sentence     comprehension at all linguistic levels (e.g. lexical,
+    syntactic, discourse) as well as higher-level text comprehension.
+
+    Check the respective `repository <https://osf.io/dn5hp/>`_ for details.
+
+    Attributes
+    ----------
+    name : str
+        The name of the dataset.
+
+    mirrors : tuple[str, ...]
+        A tuple of mirrors of the dataset. Each entry must be of type `str` and end with a '/'.
+
+    resources : tuple[dict[str, str], ...]
+        A tuple of dataset resources. Each list entry must be a dictionary with the following keys:
+        - `resource`: The url suffix of the resource. This will be concatenated with the mirror.
+        - `filename`: The filename under which the file is saved as.
+        - `md5`: The MD5 checksum of the respective file.
+
+    experiment : Experiment
+        The experiment definition.
+
+    filename_format : str
+        Regular expression which will be matched before trying to load the file. Namedgroups will
+        appear in the `fileinfo` dataframe.
+
+    filename_format_dtypes : dict[str, type], optional
+        If named groups are present in the `filename_format`, this makes it possible to cast
+        specific named groups to a particular datatype.
+
+    column_map : dict[str, str]
+        The keys are the columns to read, the values are the names to which they should be renamed.
+
+    custom_read_kwargs : dict[str, Any], optional
+        If specified, these keyword arguments will be passed to the file reading function.
+
+    Examples
+    --------
+    Initialize your :py:class:`~pymovements.PublicDataset` object with the
+    :py:class:`~pymovements.PoTeC` definition:
+
+    >>> import pymovements as pm
+    >>>
+    >>> dataset = pm.Dataset("PoTeC", path='data/PoTeC')
+
+    Download the dataset resources resources:
+
+    >>> dataset.download()# doctest: +SKIP
+
+    Load the data into memory:
+
+    >>> dataset.load()# doctest: +SKIP
+    """
+
+    # pylint: disable=similarities
+    # The PublicDatasetDefinition child classes potentially share code chunks for definitions.
+
+    name: str = 'PoTeC'
+
+    mirrors: tuple[str, ...] = (
+        'https://osf.io/download/',
+    )
+
+    resources: tuple[dict[str, str], ...] = (
+        {
+            'resource': 'tgd9q/',
+            'filename': 'PoTeC.zip',
+            'md5': '7780904bf7b18ba7d30a811174750db3',
+        },
+    )
+
+    experiment: Experiment = Experiment(
+        screen_width_px=1680,
+        screen_height_px=1050,
+        screen_width_cm=47.5,
+        screen_height_cm=30,
+        distance_cm=65,
+        origin='lower left',
+        sampling_rate=1000,
+    )
+
+    filename_format: str = r'reader{subject_id:d}_{text_id}_raw_data.tsv'
+
+    filename_format_dtypes: dict[str, type] = field(
+        default_factory=lambda: {
+            'subject_id': int,
+            'text_id': str,
+        },
+    )
+
+    trial_columns: list[str] = field(
+        default_factory=lambda: ['subject_id', 'text_id'],
+    )
+
+    time_column: str = 'time'
+    pixel_columns: list[str] = field(
+        default_factory=lambda: [
+            'x', 'y',
+        ],
+    )
+
+    custom_read_kwargs: dict[str, Any] = field(
+        default_factory=lambda: {
+            'dtypes': {
+                'time': pl.Int64,
+                'x': pl.Float32,
+                'y': pl.Float32,
+                'pupil_diameter': pl.Float32,
+            },
+            'separator': '\t',
+        },
+    )


Test patch:
diff --git a/tests/files/potec_example.tsv b/tests/files/potec_example.tsv
new file mode 100644
index 0000000..c7e7f37
--- /dev/null
+++ b/tests/files/potec_example.tsv
@@ -0,0 +1,11 @@
+time	x	y	pupil_diameter
+2076105	91.8	67.3	1746.0
+2076106	91.7	65.8	1745.0
+2076107	91.7	64.3	1743.0
+2076108	92.2	62.9	1739.0
+2076109	92.7	62.2	1739.0
+2076110	92.9	63.0	1738.0
+2076111	92.4	63.7	1738.0
+2076112	92.1	63.5	1740.0
+2076113	91.7	63.4	1743.0
+2076114	91.5	64.8	1746.0
diff --git a/tests/functional/dataset_processing_test.py b/tests/functional/dataset_processing_test.py
index fa44287..dd1fd9d 100644
--- a/tests/functional/dataset_processing_test.py
+++ b/tests/functional/dataset_processing_test.py
@@ -38,6 +38,7 @@ import pymovements as pm
         'gazebase_vr',
         'gazegraph',
         'judo1000',
+        'potec',
     ],
 )
 def fixture_dataset_init_kwargs(request):
@@ -98,6 +99,11 @@ def fixture_dataset_init_kwargs(request):
             filename_format_dtypes={},
             trial_columns=['trial_id'],
         ),
+        'potec': pm.datasets.PoTeC(
+            filename_format='potec_example.tsv',
+            filename_format_dtypes={},
+            trial_columns=None,
+        ),
 
     }
     yield pm.dataset.Dataset(
diff --git a/tests/functional/gaze_file_processing_test.py b/tests/functional/gaze_file_processing_test.py
index 2379567..d3921a6 100644
--- a/tests/functional/gaze_file_processing_test.py
+++ b/tests/functional/gaze_file_processing_test.py
@@ -39,6 +39,7 @@ import pymovements as pm
         'gazebase',
         'gazebase_vr',
         'judo1000',
+        'potec',
     ],
 )
 def fixture_gaze_init_kwargs(request):
@@ -106,6 +107,13 @@ def fixture_gaze_init_kwargs(request):
             'experiment': pm.datasets.JuDo1000().experiment,
             **pm.datasets.JuDo1000().custom_read_kwargs,
         },
+        'potec': {
+            'file': 'tests/files/potec_example.tsv',
+            'time_column': pm.datasets.PoTeC().time_column,
+            'pixel_columns': pm.datasets.PoTeC().pixel_columns,
+            'experiment': pm.datasets.PoTeC().experiment,
+            **pm.datasets.PoTeC().custom_read_kwargs,
+        },
 
     }
     yield init_param_dict[request.param]
@@ -115,7 +123,7 @@ def test_gaze_file_processing(gaze_from_kwargs):
     # Load in gaze file.
     file_extension = os.path.splitext(gaze_from_kwargs['file'])[1]
     gaze = None
-    if file_extension in {'.txt', '.csv'}:
+    if file_extension in {'.txt', '.csv', '.tsv', '.txt'}:
         gaze = pm.gaze.from_csv(**gaze_from_kwargs)
     elif file_extension in {'.feather', '.ipc'}:
         gaze = pm.gaze.from_ipc(**gaze_from_kwargs)
diff --git a/tests/unit/datasets/datasets_test.py b/tests/unit/datasets/datasets_test.py
index 4d12e5d..d19d63b 100644
--- a/tests/unit/datasets/datasets_test.py
+++ b/tests/unit/datasets/datasets_test.py
@@ -34,6 +34,7 @@ import pymovements as pm
         pytest.param(pm.datasets.GazeOnFaces, 'GazeOnFaces', id='GazeOnFaces'),
         pytest.param(pm.datasets.HBN, 'HBN', id='HBN'),
         pytest.param(pm.datasets.JuDo1000, 'JuDo1000', id='JuDo1000'),
+        pytest.param(pm.datasets.PoTeC, 'PoTeC', id='PoTeC'),
         pytest.param(pm.datasets.SBSAT, 'SBSAT', id='SBSAT'),
     ],
 )
@@ -52,6 +53,7 @@ def test_public_dataset_registered(definition_class, dataset_name):
         pytest.param(pm.datasets.GazeOnFaces, id='GazeOnFaces'),
         pytest.param(pm.datasets.HBN, id='HBN'),
         pytest.param(pm.datasets.JuDo1000, id='JuDo1000'),
+        pytest.param(pm.datasets.PoTeC, id='PoTeC'),
         pytest.param(pm.datasets.SBSAT, id='SBSAT'),
     ],
 )
diff --git a/tests/unit/datasets/potec_test.py b/tests/unit/datasets/potec_test.py
new file mode 100644
index 0000000..38b72d4
--- /dev/null
+++ b/tests/unit/datasets/potec_test.py
@@ -0,0 +1,79 @@
+# Copyright (c) 2023 The pymovements Project Authors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""Test all functionality in pymovements.dataset.potec."""
+from pathlib import Path
+
+import pytest
+
+import pymovements as pm
+
+
+@pytest.mark.parametrize(
+    ('init_path', 'expected_paths'),
+    [
+        pytest.param(
+            '/data/set/path',
+            {
+                'root': Path('/data/set/path/'),
+                'dataset': Path('/data/set/path/'),
+                'download': Path('/data/set/path/downloads'),
+            },
+        ),
+        pytest.param(
+            pm.DatasetPaths(root='/data/set/path'),
+            {
+                'root': Path('/data/set/path/'),
+                'dataset': Path('/data/set/path/PoTeC'),
+                'download': Path('/data/set/path/PoTeC/downloads'),
+            },
+        ),
+        pytest.param(
+            pm.DatasetPaths(root='/data/set/path', dataset='.'),
+            {
+                'root': Path('/data/set/path/'),
+                'dataset': Path('/data/set/path/'),
+                'download': Path('/data/set/path/downloads'),
+            },
+        ),
+        pytest.param(
+            pm.DatasetPaths(root='/data/set/path', dataset='dataset'),
+            {
+                'root': Path('/data/set/path/'),
+                'dataset': Path('/data/set/path/dataset'),
+                'download': Path('/data/set/path/dataset/downloads'),
+            },
+        ),
+        pytest.param(
+            pm.DatasetPaths(root='/data/set/path', downloads='custom_downloads'),
+            {
+                'root': Path('/data/set/path/'),
+                'dataset': Path('/data/set/path/PoTeC'),
+                'download': Path('/data/set/path/PoTeC/custom_downloads'),
+            },
+        ),
+    ],
+)
+def test_paths(init_path, expected_paths):
+    dataset = pm.Dataset(pm.datasets.PoTeC, path=init_path)
+
+    assert dataset.paths.root == expected_paths['root']
+    assert dataset.path == expected_paths['dataset']
+    assert dataset.paths.dataset == expected_paths['dataset']
+    assert dataset.paths.downloads == expected_paths['download']