Repository: aeye-lab/pymovements (python)

Issue:
Unexpected unnest-sideeffect in Dataset.save_preprocessed()
## Current Behavior
When calling `Dataset.save_preprocessed()` with `extension='csv'` columns need to be unnested before saving.
This part is definitely intended. But it is unexpected that the columns stay unnested after the call.

The user now needs to call `nest` again if they want to run preprocessing functions.

## Expected Behavior
The saving of the dataset should not alter any dataframe.
The unnesting should be done on a copy during saving.

## Minimum acceptance criteria
- [ ] no side-effects on GazeDataFrame during saving.


Pull request:
fix: Get rid of unnest side effect during GazeDataFrame.save()
## Description

Closes #480 

## Implemented changes

The gaze dataframe will now be completely copied before post-processing and saving.

## Type of change

- [x] Bug fix (non-breaking change which fixes an issue)

## How Has This Been Tested?

- [x] I have written a test for checking that the saved gaze dataframe is exactly equal (failed before my changes)

## Checklist:

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my feature works
- [x] New and existing unit tests pass locally with my changes
- [x] Any dependent changes have been merged and published in downstream modules
- [x] I have checked my code and corrected any misspellings


Patch:
diff --git a/src/pymovements/dataset/dataset_files.py b/src/pymovements/dataset/dataset_files.py
index 949f726..cf2ba4a 100644
--- a/src/pymovements/dataset/dataset_files.py
+++ b/src/pymovements/dataset/dataset_files.py
@@ -467,6 +467,8 @@ def save_preprocessed(
     disable_progressbar = not verbose
 
     for file_id, gaze_df in enumerate(tqdm(gaze, disable=disable_progressbar)):
+        gaze_df = gaze_df.copy()
+
         raw_filepath = paths.raw / Path(fileinfo[file_id, 'filepath'])
         preprocessed_filepath = paths.get_preprocessed_filepath(
             raw_filepath, preprocessed_dirname=preprocessed_dirname,
@@ -486,19 +488,18 @@ def save_preprocessed(
             if 'acceleration' in gaze_df.frame.columns:
                 gaze_df.unnest('acceleration')
 
-        gaze_df_out = gaze_df.frame.clone()
         for column in gaze_df.columns:
             if column in fileinfo.columns:
-                gaze_df_out = gaze_df_out.drop(column)
+                gaze_df.frame = gaze_df.frame.drop(column)
 
         if verbose >= 2:
             print('Save file to', preprocessed_filepath)
 
         preprocessed_filepath.parent.mkdir(parents=True, exist_ok=True)
         if extension == 'feather':
-            gaze_df_out.write_ipc(preprocessed_filepath)
+            gaze_df.frame.write_ipc(preprocessed_filepath)
         elif extension == 'csv':
-            gaze_df_out.write_csv(preprocessed_filepath)
+            gaze_df.frame.write_csv(preprocessed_filepath)
         else:
             valid_extensions = ['csv', 'feather']
             raise ValueError(
diff --git a/src/pymovements/gaze/gaze_dataframe.py b/src/pymovements/gaze/gaze_dataframe.py
index 7e32222..8d1a0e0 100644
--- a/src/pymovements/gaze/gaze_dataframe.py
+++ b/src/pymovements/gaze/gaze_dataframe.py
@@ -470,10 +470,12 @@ class GazeDataFrame:
         GazeDataFrame
             A copy of the GazeDataFrame.
         """
-        return GazeDataFrame(
+        gaze = GazeDataFrame(
             data=self.frame.clone(),
             experiment=deepcopy(self.experiment),
         )
+        gaze.n_components = self.n_components
+        return gaze
 
     def _check_experiment(self) -> None:
         """Check if experiment attribute has been set."""


Test patch:
diff --git a/tests/dataset/dataset_test.py b/tests/dataset/dataset_test.py
index c45bc86..9e2f99c 100644
--- a/tests/dataset/dataset_test.py
+++ b/tests/dataset/dataset_test.py
@@ -1026,19 +1026,24 @@ def test_save_preprocessed_directory_exists(
     )
 
 
-def test_save_preprocessed_no_tuple_columns(dataset_configuration):
+@pytest.mark.parametrize(
+    'drop_column',
+    [
+        'time',
+        'pixel',
+        'position',
+        'velocity',
+        'acceleration',
+    ],
+)
+def test_save_preprocessed(dataset_configuration, drop_column):
     dataset = pm.Dataset(**dataset_configuration['init_kwargs'])
     dataset.load()
     dataset.pix2deg()
     dataset.pos2vel()
     dataset.pos2acc()
 
-    # This is not implemented yet
-    # dataset.gaze[0].unnest(['pixel', 'position', 'velocity', 'acceleration'])
-    dataset.gaze[0].frame = dataset.gaze[0].frame.rename({'time': 'ttt'})
-    dataset.gaze[0].frame = dataset.gaze[0].frame.drop(
-        ['pixel', 'position', 'velocity', 'acceleration'],
-    )
+    dataset.gaze[0].frame = dataset.gaze[0].frame.drop(drop_column)
 
     preprocessed_dirname = 'preprocessed-test'
     shutil.rmtree(dataset.path / Path(preprocessed_dirname), ignore_errors=True)
@@ -1051,6 +1056,37 @@ def test_save_preprocessed_no_tuple_columns(dataset_configuration):
     )
 
 
+@pytest.mark.parametrize(
+    'drop_column',
+    [
+        'time',
+        'pixel',
+        'position',
+        'velocity',
+        'acceleration',
+    ],
+)
+def test_save_preprocessed_has_no_side_effect(dataset_configuration, drop_column):
+    dataset = pm.Dataset(**dataset_configuration['init_kwargs'])
+    dataset.load()
+    dataset.pix2deg()
+    dataset.pos2vel()
+    dataset.pos2acc()
+
+    dataset.gaze[0].frame = dataset.gaze[0].frame.drop(drop_column)
+
+    old_frame = dataset.gaze[0].frame.clone()
+
+    preprocessed_dirname = 'preprocessed-test'
+    shutil.rmtree(dataset.path / Path(preprocessed_dirname), ignore_errors=True)
+    shutil.rmtree(dataset.path / Path(preprocessed_dirname), ignore_errors=True)
+    dataset.save_preprocessed(preprocessed_dirname, extension='csv')
+
+    new_frame = dataset.gaze[0].frame.clone()
+
+    assert_frame_equal(old_frame, new_frame)
+
+
 @pytest.mark.parametrize(
     ('expected_save_preprocessed_path', 'expected_save_events_path', 'save_kwargs'),
     [