Repository: aeye-lab/pymovements
Pull request #801 (2024-09-16 09:21:39)

add didec dataset and tests

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/aeye-lab/pymovements/pull/801?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=aeye-lab) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 100.00%. Comparing base [(`5868d11`)](https://app.codecov.io/gh/aeye-lab/pymovements/commit/5868d11f136930cb04212075daee3b942309f6fa?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=aeye-lab) to head [(`45e082f`)](https://app.codecov.io/gh/aeye-lab/pymovements/commit/45e082fed1f24163e7e3b10c6b801c8b792fe677?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=aeye-lab).

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff            @@
##              main      #801   +/-   ##
=========================================
  Coverage   100.00%   100.00%           
=========================================
  Files           66        67    +1     
  Lines         3055      3080   +25     
  Branches       871       883   +12     
=========================================
+ Hits          3055      3080   +25     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/aeye-lab/pymovements/pull/801?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=aeye-lab).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=aeye-lab).


Patch:
diff --git a/docs/source/bibliography.bib b/docs/source/bibliography.bib
index e117c78..0603bb6 100644
--- a/docs/source/bibliography.bib
+++ b/docs/source/bibliography.bib
@@ -63,6 +63,24 @@
     pages = "13586--13594",
 }
 
+@inproceedings{DIDEC,
+    title = "{DIDEC}: The {D}utch Image Description and Eye-tracking Corpus",
+    author = "van Miltenburg, Emiel  and
+      K{\'a}d{\'a}r, {\'A}kos  and
+      Koolen, Ruud  and
+      Krahmer, Emiel",
+    editor = "Bender, Emily M.  and
+      Derczynski, Leon  and
+      Isabelle, Pierre",
+    booktitle = "Proceedings of the 27th International Conference on Computational Linguistics",
+    month = aug,
+    year = "2018",
+    address = "Santa Fe, New Mexico, USA",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/C18-1310",
+    pages = "3658--3669",
+}
+
 @misc{EMTeC,
       title={EMTeC: A Corpus of Eye Movements on Machine-Generated Texts},
       author={Lena Sophia Bolliger and Patrick Haller and Isabelle Caroline Rose Cretton and David Robert Reich and Tannon Kew and Lena Ann Jäger},
diff --git a/src/pymovements/datasets/__init__.py b/src/pymovements/datasets/__init__.py
index 6928718..8521fe5 100644
--- a/src/pymovements/datasets/__init__.py
+++ b/src/pymovements/datasets/__init__.py
@@ -26,6 +26,7 @@
    :template: class.rst
 
     pymovements.datasets.CopCo
+    pymovements.datasets.DIDEC
     pymovements.datasets.EMTeC
     pymovements.datasets.GazeBase
     pymovements.datasets.GazeBaseVR
@@ -47,6 +48,7 @@
     pymovements.datasets.ToyDatasetEyeLink
 """
 from pymovements.datasets.copco import CopCo
+from pymovements.datasets.didec import DIDEC
 from pymovements.datasets.emtec import EMTeC
 from pymovements.datasets.gaze_graph import GazeGraph
 from pymovements.datasets.gaze_on_faces import GazeOnFaces
@@ -62,6 +64,7 @@ from pymovements.datasets.toy_dataset_eyelink import ToyDatasetEyeLink
 
 __all__ = [
     'CopCo',
+    'DIDEC',
     'EMTeC',
     'GazeBase',
     'GazeBaseVR',
diff --git a/src/pymovements/datasets/didec.py b/src/pymovements/datasets/didec.py
new file mode 100644
index 0000000..6c64c52
--- /dev/null
+++ b/src/pymovements/datasets/didec.py
@@ -0,0 +1,202 @@
+# Copyright (c) 2022-2024 The pymovements Project Authors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""Provides a definition for the DIDEC dataset."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Any
+
+from pymovements.dataset.dataset_definition import DatasetDefinition
+from pymovements.dataset.dataset_library import register_dataset
+from pymovements.gaze.experiment import Experiment
+
+
+@dataclass
+@register_dataset
+class DIDEC(DatasetDefinition):
+    """DIDEC dataset :cite:p:`DIDEC`.
+
+    The DIDEC eye-tracking data has two different data collections, (1) for the
+    description viewing task is more coherent than for the free-viewing task;
+    (2) variation in image descriptions. The data was collected using BeGaze eye-tracker
+    with a sampling rate of 250 Hz. The data collection contains 112 Dutch students,
+    54 students completed the free viewing task, while 58 completed the image description task.
+
+    Check the respective paper for details :cite:p:`DIDEC`.
+
+    Attributes
+    ----------
+    name: str
+        The name of the dataset.
+
+    mirrors: dict[str, tuple[str, ...]]
+        A tuple of mirrors of the dataset. Each entry must be of type `str` and end with a '/'.
+
+    resources: dict[str, tuple[dict[str, str], ...]]
+        A tuple of dataset resources. Each list entry must be a dictionary with the following keys:
+        - `resource`: The url suffix of the resource. This will be concatenated with the mirror.
+        - `filename`: The filename under which the file is saved as.
+        - `md5`: The MD5 checksum of the respective file.
+
+    experiment: Experiment
+        The experiment definition.
+
+    filename_format: dict[str, str]
+        Regular expression which will be matched before trying to load the file. Namedgroups will
+        appear in the `fileinfo` dataframe.
+
+    filename_format_dtypes: dict[str, dict[str, type]]
+        If named groups are present in the `filename_format`, this makes it possible to cast
+        specific named groups to a particular datatype.
+
+    column_map: dict[str, str]
+        The keys are the columns to read, the values are the names to which they should be renamed.
+
+    custom_read_kwargs: dict[str, dict[str, Any]]
+        If specified, these keyword arguments will be passed to the file reading function.
+
+    Examples
+    --------
+    Initialize your :py:class:`~pymovements.PublicDataset` object with the
+    :py:class:`~pymovements.DIDEC` definition:
+
+    >>> import pymovements as pm
+    >>>
+    >>> dataset = pm.Dataset("DIDEC", path='data/DIDEC')
+
+    Download the dataset resources:
+
+    >>> dataset.download()# doctest: +SKIP
+
+    Load the data into memory:
+
+    >>> dataset.load()# doctest: +SKIP
+    """
+
+    # pylint: disable=similarities
+    # The PublicDatasetDefinition child classes potentially share code chunks for definitions.
+
+    name: str = 'DIDEC'
+
+    has_files: dict[str, bool] = field(
+        default_factory=lambda: {'gaze': True, 'precomputed_events': False},
+    )
+
+    mirrors: dict[str, tuple[str, ...]] = field(
+        default_factory=lambda: {
+            'gaze': ('https://didec.uvt.nl/corpus/',),
+        },
+    )
+
+    resources: dict[str, tuple[dict[str, str], ...]] = field(
+        default_factory=lambda: {
+            'gaze': (
+                {
+                    'resource': 'DIDEC_only_the_eyetracking_data.zip',
+                    'filename': 'DIDEC_only_the_eyetracking_data.zip',
+                    'md5': 'd572b0b41828986ca48a2fcf6966728a',
+                },
+            ),
+        },
+    )
+
+    extract: dict[str, bool] = field(default_factory=lambda: {'gaze': True})
+
+    experiment: Experiment = Experiment(
+        screen_width_px=1680,
+        screen_height_px=1050,
+        screen_width_cm=47.4,
+        screen_height_cm=29.7,
+        distance_cm=70,
+        origin='upper left',
+        sampling_rate=1000,
+    )
+
+    filename_format: dict[str, str] = field(
+        default_factory=lambda: {
+            'gaze':
+                (
+                    r'Ruud_exp{experiment:d}_'
+                    r'list{list:d}_v{version:d}_'
+                    r'ppn{participant:d}_{session:d}_'
+                    r'Trial{trial:d} Samples.txt'
+                ),
+        },
+    )
+
+    filename_format_dtypes: dict[str, dict[str, type]] = field(
+        default_factory=lambda: {
+            'gaze': {
+                'experiment': int,
+                'list': int,
+                'version': int,
+                'participant': int,
+                'session': int,
+                'trial': int,
+            },
+        },
+    )
+
+    trial_columns: list[str] = field(
+        default_factory=lambda: ['Stimulus'],
+    )
+
+    time_column: str = 'Time'
+
+    time_unit: str = 'ms'
+
+    pixel_columns: list[str] = field(
+        default_factory=lambda: [
+            'L POR X [px]',
+            'L POR Y [px]',
+            'R POR X [px]',
+            'R POR Y [px]',
+        ],
+    )
+
+    column_map: dict[str, str] = field(default_factory=lambda: {})
+
+    custom_read_kwargs: dict[str, dict[str, Any]] = field(
+        default_factory=lambda: {
+            'gaze': {
+                'separator': '\t',
+                # skip begaze tracker data
+                'skip_rows': 43,
+                'has_header': False,
+                'new_columns': [
+                    'Time',
+                    'Type',
+                    'Trial',
+                    'L POR X [px]',
+                    'L POR Y [px]',
+                    'R POR X [px]',
+                    'R POR Y [px]',
+                    'Timing',
+                    'Pupil Confidence',
+                    'L Plane',
+                    'R Plane',
+                    'L Event Info',
+                    'R Event Info',
+                    'Stimulus',
+                ],
+            },
+        },
+    )


Test patch:
diff --git a/tests/files/didec_example.txt b/tests/files/didec_example.txt
new file mode 100644
index 0000000..2a1eb7c
--- /dev/null
+++ b/tests/files/didec_example.txt
@@ -0,0 +1,58 @@
+##  adjust example file
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+##
+Time	Type	Trial	L POR X [px]	L POR Y [px]	R POR X [px]	R POR Y [px]	Timing	Pupil Confidence	L Plane	R Plane	Frame	Aux1	L Event Info	R Event Info	Stimulus
+70590724309	MSG	1	# Message: test.bmp
+70590726108	SMP	1	858.16	516.11	858.16	516.11	0	3	1	1	-1		-	-	test.bmp
+70590730097	SMP	1	844.17	517.15	844.17	517.15	0	3	1	1	-1		-	-	test.bmp
+70590734091	SMP	1	869.13	528.14	869.13	528.14	0	3	1	1	-1		-	-	test.bmp
+70590738188	SMP	1	849.13	529.17	849.13	529.17	0	3	1	1	-1		-	-	test.bmp
+70590742154	SMP	1	843.17	535.12	843.17	535.12	0	3	1	1	-1		-	-	test.bmp
+70590746235	SMP	1	810.13	526.17	810.13	526.17	0	3	1	1	-1		-	-	test.bmp
+70590750155	SMP	1	836.15	533.18	836.15	533.18	0	3	1	1	-1		-	-	test.bmp
+70590754105	SMP	1	826.17	533.16	826.17	533.16	0	3	1	1	-1		-	-	test.bmp
+70590758122	SMP	1	837.12	533.17	837.12	533.17	0	3	1	1	-1		-	-	test.bmp
+70590762099	SMP	1	830.11	537.13	830.11	537.13	0	3	1	1	-1		-	-	test.bmp
+70590766124	SMP	1	828.18	534.12	828.18	534.12	0	3	1	1	-1		-	-	test.bmp
+70590770081	SMP	1	825.18	538.11	825.18	538.11	0	3	1	1	-1		-	-	test.bmp
+70590774123	SMP	1	820.15	535.18	820.15	535.18	0	3	1	1	-1		-	-	test.bmp
+70590778158	SMP	1	822.10	536.13	822.10	536.13	0	3	1	1	-1		-	-	test.bmp
+70590782136	SMP	1	821.12	533.19	821.12	533.19	0	3	1	1	-1		Saccade	Saccade	test.bmp
diff --git a/tests/functional/dataset_processing_test.py b/tests/functional/dataset_processing_test.py
index 6c35f6e..2b3dd05 100644
--- a/tests/functional/dataset_processing_test.py
+++ b/tests/functional/dataset_processing_test.py
@@ -30,6 +30,7 @@ import pymovements as pm
         'csv_binocular',
         'ipc_monocular',
         'ipc_binocular',
+        'didec',
         'hbn',
         'sbsat',
         'gaze_on_faces',
@@ -77,6 +78,14 @@ def fixture_dataset_init_kwargs(request):
             filename_format_dtypes={'gaze': {}},
             custom_read_kwargs={'gaze': {}},
         ),
+        'didec': pm.datasets.DIDEC(
+            has_files={'gaze': True, 'precomputed_events': False},
+            filename_format={'gaze': 'didec_example.txt'},
+            time_column=pm.datasets.DIDEC().time_column,
+            time_unit=pm.datasets.DIDEC().time_unit,
+            filename_format_dtypes={'gaze': {}},
+            trial_columns=None,
+        ),
         'hbn': pm.datasets.HBN(
             has_files={'gaze': True, 'precomputed_events': False},
             filename_format={'gaze': 'hbn_example.csv'},
diff --git a/tests/functional/gaze_file_processing_test.py b/tests/functional/gaze_file_processing_test.py
index 5a351b3..68d5961 100644
--- a/tests/functional/gaze_file_processing_test.py
+++ b/tests/functional/gaze_file_processing_test.py
@@ -33,6 +33,7 @@ import pymovements as pm
         'ipc_monocular',
         'ipc_binocular',
         'eyelink_monocular',
+        'didec',
         'hbn',
         'sbsat',
         'gaze_on_faces',
@@ -71,6 +72,14 @@ def fixture_gaze_init_kwargs(request):
             'file': 'tests/files/eyelink_monocular_example.asc',
             'experiment': pm.datasets.ToyDatasetEyeLink().experiment,
         },
+        'didec': {
+            'file': 'tests/files/didec_example.txt',
+            'time_column': pm.datasets.DIDEC().time_column,
+            'time_unit': pm.datasets.DIDEC().time_unit,
+            'pixel_columns': pm.datasets.DIDEC().pixel_columns,
+            'experiment': pm.datasets.DIDEC().experiment,
+            **pm.datasets.DIDEC().custom_read_kwargs['gaze'],
+        },
         'hbn': {
             'file': 'tests/files/hbn_example.csv',
             'time_column': pm.datasets.HBN().time_column,
diff --git a/tests/unit/datasets/datasets_test.py b/tests/unit/datasets/datasets_test.py
index bec03e5..1110a1d 100644
--- a/tests/unit/datasets/datasets_test.py
+++ b/tests/unit/datasets/datasets_test.py
@@ -32,6 +32,7 @@ import pymovements as pm
     # XXX: add public dataset in alphabetical order
     [
         pytest.param(pm.datasets.CopCo, 'CopCo', id='CopCo'),
+        pytest.param(pm.datasets.DIDEC, 'DIDEC', id='DIDEC'),
         pytest.param(pm.datasets.EMTeC, 'EMTeC', id='EMTeC'),
         pytest.param(pm.datasets.GazeBase, 'GazeBase', id='GazeBase'),
         pytest.param(pm.datasets.GazeBaseVR, 'GazeBaseVR', id='GazeBaseVR'),