Repository: spikeinterface/spikeinterface
Issue: Portability issue: hard-coded paths used for WaveformExtractor.load_from_folder

I believe this is a distinct issue, although it is at least a cousin to #422

The scenario is using spikeinterface 0.92.0 and mountainsort 1.0.0. Following a successful first-time run of these commands, the waveform extract directory is created.

```
sorting_MS4 = ss.run_mountainsort4(r, output_folder=output_dir, verbose=False, **params)
we = si.extract_waveforms(r, sorting_MS4, output_dir + '_waveforms', load_if_exists=True, ms_before=1, ms_after=2, 
                          max_spikes_per_unit=500, n_jobs=1, chunk_size=30000)
```

The issue here is that the `recording.json` and the `sorting.json` files in the extraction directory refer to full paths for the raw data and the sorting results, respectively. The result is that using the sorting/extraction results are not portable, and `extract_waveforms` needs to be run in the exact original environment.

I've seen this in two cases:

* moving sorting & extraction results to a different machine (without raw data)
* (example in traceback) trying to load a `WaveformExtractor` in a new python process or Docker container after sorting from data saved to a temporary file (e.g. after `NumpyRecording.save`)

Secondarily, does `WaveformExtractor` need a path on the raw data at all?

Right now I need to shim the json files before I can load the results. I was digging a little before I got busy, but maybe relative paths are supported?


traceback (note that the `None` argument for recording makes no difference)

```python
---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
/tmp/ipykernel_116/923429382.py in <module>
----> 1 we = si.extract_waveforms(None, sorting_MS4, output_dir + '_waveforms', load_if_exists=True, ms_before=1, ms_after=2, 
      2                           max_spikes_per_unit=500, n_jobs=20, chunk_size=30000)

/opt/conda/envs/msort/lib/python3.8/site-packages/spikeinterface/core/waveform_extractor.py in extract_waveforms(recording, sorting, folder, load_if_exists, precompute_template, ms_before, ms_after, max_spikes_per_unit, overwrite, return_scaled, dtype, **job_kwargs)
    683         shutil.rmtree(folder)
    684     if load_if_exists and folder.is_dir():
--> 685         we = WaveformExtractor.load_from_folder(folder)
    686     else:
    687         we = WaveformExtractor.create(recording, sorting, folder)

/opt/conda/envs/msort/lib/python3.8/site-packages/spikeinterface/core/waveform_extractor.py in load_from_folder(cls, folder)
     90         folder = Path(folder)
     91         assert folder.is_dir(), f'This folder do not exists {folder}'
---> 92         recording = load_extractor(folder / 'recording.json')
     93         sorting = load_extractor(folder / 'sorting.json')
     94         we = cls(recording, sorting, folder)

/opt/conda/envs/msort/lib/python3.8/site-packages/spikeinterface/core/base.py in load_extractor(file_or_folder_or_dict, base_folder)
    755         return BaseExtractor.from_dict(file_or_folder_or_dict, base_folder=base_folder)
    756     else:
--> 757         return BaseExtractor.load(file_or_folder_or_dict, base_folder=base_folder)
    758 
    759 

/opt/conda/envs/msort/lib/python3.8/site-packages/spikeinterface/core/base.py in load(file_path, base_folder)
    472             else:
    473                 raise ValueError(f'Impossible to load {file_path}')
--> 474             extractor = BaseExtractor.from_dict(d, base_folder=base_folder)
    475             return extractor
    476 

/opt/conda/envs/msort/lib/python3.8/site-packages/spikeinterface/core/base.py in from_dict(d, base_folder)
    301             assert base_folder is not None, 'When  relative_paths=True, need to provide base_folder'
    302             d = _make_paths_absolute(d, base_folder)
--> 303         extractor = _load_extractor_from_dict(d)
    304         folder_metadata = d.get('folder_metadata', None)
    305         if folder_metadata is not None:

/opt/conda/envs/msort/lib/python3.8/site-packages/spikeinterface/core/base.py in _load_extractor_from_dict(dic)
    684 
    685         if isinstance(v, dict) and is_dict_extractor(v):
--> 686             kwargs[k] = _load_extractor_from_dict(v)
    687 
    688             # handle list of extractors list

/opt/conda/envs/msort/lib/python3.8/site-packages/spikeinterface/core/base.py in _load_extractor_from_dict(dic)
    684 
    685         if isinstance(v, dict) and is_dict_extractor(v):
--> 686             kwargs[k] = _load_extractor_from_dict(v)
    687 
    688             # handle list of extractors list

/opt/conda/envs/msort/lib/python3.8/site-packages/spikeinterface/core/base.py in _load_extractor_from_dict(dic)
    701 
    702     # instantiate extrator object
--> 703     extractor = cls(**kwargs)
    704 
    705     extractor._annotations.update(dic['annotations'])

/opt/conda/envs/msort/lib/python3.8/site-packages/spikeinterface/core/binaryrecordingextractor.py in __init__(self, file_paths, sampling_frequency, num_chan, dtype, t_starts, channel_ids, time_axis, file_offset, gain_to_uV, offset_to_uV, is_filtered)
     81             else:
     82                 t_start = t_starts[i]
---> 83             rec_segment = BinaryRecordingSegment(datfile, sampling_frequency, t_start, num_chan, dtype, time_axis, file_offset)
     84             self.add_recording_segment(rec_segment)
     85 

/opt/conda/envs/msort/lib/python3.8/site-packages/spikeinterface/core/binaryrecordingextractor.py in __init__(self, datfile, sampling_frequency, t_start, num_chan, dtype, time_axis, file_offset)
    127     def __init__(self, datfile, sampling_frequency, t_start, num_chan, dtype, time_axis, file_offset):
    128         BaseRecordingSegment.__init__(self, sampling_frequency=sampling_frequency, t_start=t_start)
--> 129         self._timeseries = read_binary_recording(datfile, num_chan, dtype, time_axis, file_offset)
    130 
    131     def get_num_samples(self) -> int:

/opt/conda/envs/msort/lib/python3.8/site-packages/spikeinterface/core/core_tools.py in read_binary_recording(file, num_chan, dtype, time_axis, offset)
    138     '''
    139     num_chan = int(num_chan)
--> 140     with Path(file).open() as f:
    141         nsamples = (os.fstat(f.fileno()).st_size - offset) // (num_chan * np.dtype(dtype).itemsize)
    142     if time_axis == 0:

/opt/conda/envs/msort/lib/python3.8/pathlib.py in open(self, mode, buffering, encoding, errors, newline)
   1220         if self._closed:
   1221             self._raise_closed()
-> 1222         return io.open(self, mode, buffering, encoding, errors, newline,
   1223                        opener=self._opener)
   1224 

/opt/conda/envs/msort/lib/python3.8/pathlib.py in _opener(self, name, flags, mode)
   1076     def _opener(self, name, flags, mode=0o666):
   1077         # A stub for the opener argument to built-in open()
-> 1078         return self._accessor.open(self, flags, mode)
   1079 
   1080     def _raw_open(self, flags, mode=0o777):

FileNotFoundError: [Errno 2] No such file or directory: '/tmp/spikeinterface_cache/tmpviuk8wbl/OCJYFCHA/traces_cached_seg0.raw'
```

Changed files:
spikeinterface/core/waveform_extractor.py, spikeinterface/core/tests/test_waveform_extractor.py, spikeinterface/core/base.py

diff --git a/spikeinterface/core/base.py b/spikeinterface/core/base.py
index 576f8721..f9ca3afd 100644
--- a/spikeinterface/core/base.py
+++ b/spikeinterface/core/base.py
@@ -4,6 +4,7 @@ from copy import deepcopy
 import weakref
 import json
 import pickle
+import os
 import random
 import string
 import warnings
@@ -649,12 +650,14 @@ def _make_paths_relative(d, relative):
             if "path" in k:
                 # paths can be str or list of str
                 if isinstance(d[k], str):
-                    d[k] = str(Path(d[k]).relative_to(relative))
+                    # we use os.path.relpath here to allow for relative paths with respect to shared root
+                    d[k] = os.path.relpath(str(d[k]), start=str(relative.absolute()))
                 else:
                     assert isinstance(d[k], list), "Paths can be strings or lists in kwargs"
                     relative_paths = []
                     for path in d[k]:
-                        relative_paths.append(str(Path(path).relative_to(relative)))
+                        # we use os.path.relpath here to allow for relative paths with respect to shared root
+                        relative_paths.append(os.path.relpath(str(path), start=str(relative.absolute())))
                     d[k] = relative_paths
         return d
 
diff --git a/spikeinterface/core/tests/test_waveform_extractor.py b/spikeinterface/core/tests/test_waveform_extractor.py
index 025237fc..0fa8b3f0 100644
--- a/spikeinterface/core/tests/test_waveform_extractor.py
+++ b/spikeinterface/core/tests/test_waveform_extractor.py
@@ -182,6 +182,57 @@ def test_sparsity():
         assert temp_diff.shape[-1] == len(sparsity_diff[unit])
 
 
+def test_portability():
+    durations = [30, 40]
+    sampling_frequency = 30000.
+    
+    folder_to_move = Path("original_folder")
+    if folder_to_move.is_dir():
+        shutil.rmtree(folder_to_move)
+    folder_to_move.mkdir()
+    folder_moved = Path("moved_folder")
+    if folder_moved.is_dir():
+        shutil.rmtree(folder_moved)
+    # folder_moved.mkdir()
+        
+    # 2 segments
+    num_channels = 2
+    recording = generate_recording(num_channels=num_channels, durations=durations, 
+                                   sampling_frequency=sampling_frequency)
+    recording.annotate(is_filtered=True)
+    folder_rec = folder_to_move / "rec"
+    recording = recording.save(folder=folder_rec)
+    num_units = 15
+    sorting = generate_sorting(num_units=num_units, sampling_frequency=sampling_frequency, durations=durations)
+    folder_sort = folder_to_move / "sort"
+    sorting = sorting.save(folder=folder_sort)
+
+    wf_folder = folder_to_move / "waveform_extractor"
+    if wf_folder.is_dir():
+        shutil.rmtree(wf_folder)
+
+    # save with relative paths
+    we = extract_waveforms(recording, sorting, wf_folder, use_relative_path=True)
+
+    # move all to a separate folder
+    shutil.copytree(folder_to_move, folder_moved)
+    wf_folder_moved = folder_moved / "waveform_extractor"
+    we_loaded = extract_waveforms(recording, sorting, wf_folder_moved, load_if_exists=True)
+    
+    assert we_loaded.recording is not None
+    assert we_loaded.sorting is not None
+
+    assert np.allclose(
+        we.recording.get_channel_ids(), we_loaded.recording.get_channel_ids())
+    assert np.allclose(
+        we.sorting.get_unit_ids(), we_loaded.sorting.get_unit_ids())
+
+
+    for unit in we.sorting.get_unit_ids():
+        wf = we.get_waveforms(unit_id=unit)
+        wf_loaded = we_loaded.get_waveforms(unit_id=unit)
+
+        assert np.allclose(wf, wf_loaded)
 
 
 if __name__ == '__main__':
@@ -189,3 +240,4 @@ if __name__ == '__main__':
     test_WaveformExtractor()
     test_extract_waveforms()
     test_sparsity()
+    test_portability()
diff --git a/spikeinterface/core/waveform_extractor.py b/spikeinterface/core/waveform_extractor.py
index 8fbcaf15..97ec6e95 100644
--- a/spikeinterface/core/waveform_extractor.py
+++ b/spikeinterface/core/waveform_extractor.py
@@ -86,9 +86,11 @@ class WaveformExtractor:
     @classmethod
     def load_from_folder(cls, folder):
         folder = Path(folder)
-        assert folder.is_dir(), f'This folder do not exists {folder}'
-        recording = load_extractor(folder / 'recording.json')
-        sorting = load_extractor(folder / 'sorting.json')
+        assert folder.is_dir(), f'This folder does not exists {folder}'
+        recording = load_extractor(folder / 'recording.json',
+                                   base_folder=folder)
+        sorting = load_extractor(folder / 'sorting.json',
+                                 base_folder=folder)
         we = cls(recording, sorting, folder)
 
         for mode in _possible_template_modes:
@@ -100,7 +102,8 @@ class WaveformExtractor:
         return we
 
     @classmethod
-    def create(cls, recording, sorting, folder, remove_if_exists=False):
+    def create(cls, recording, sorting, folder, remove_if_exists=False,
+               use_relative_path=False):
         folder = Path(folder)
         if folder.is_dir():
             if remove_if_exists:
@@ -108,11 +111,16 @@ class WaveformExtractor:
             else:
                 raise FileExistsError('Folder already exists')
         folder.mkdir(parents=True)
+        
+        if use_relative_path:
+            relative_to = folder
+        else:
+            relative_to = None
 
         if recording.is_dumpable:
-            recording.dump(folder / 'recording.json', relative_to=None)
+            recording.dump(folder / 'recording.json', relative_to=relative_to)
         if sorting.is_dumpable:
-            sorting.dump(folder / 'sorting.json', relative_to=None)
+            sorting.dump(folder / 'sorting.json', relative_to=relative_to)
 
         return cls(recording, sorting, folder)
 
@@ -811,6 +819,7 @@ def extract_waveforms(recording, sorting, folder,
                       overwrite=False,
                       return_scaled=True,
                       dtype=None,
+                      use_relative_path=False,
                       **job_kwargs):
     """
     Extracts waveform on paired Recording-Sorting objects.
@@ -843,7 +852,11 @@ def extract_waveforms(recording, sorting, folder,
         If True and recording has gain_to_uV/offset_to_uV properties, waveforms are converted to uV.
     dtype: dtype or None
         Dtype of the output waveforms. If None, the recording dtype is maintained.
-
+    use_relative_path: bool
+        If True, the recording and sorting paths are relative to the waveforms folder. 
+        This allows portability of the waveform folder provided that the relative paths are the same, 
+        but forces all the data files to be in the same drive.
+        Default is False.
 
     {}
 
@@ -860,7 +873,7 @@ def extract_waveforms(recording, sorting, folder,
     if load_if_exists and folder.is_dir():
         we = WaveformExtractor.load_from_folder(folder)
     else:
-        we = WaveformExtractor.create(recording, sorting, folder)
+        we = WaveformExtractor.create(recording, sorting, folder, use_relative_path=use_relative_path)
         we.set_params(ms_before=ms_before, ms_after=ms_after, max_spikes_per_unit=max_spikes_per_unit, dtype=dtype,
                       return_scaled=return_scaled)
         we.run_extract_waveforms(**job_kwargs)