Repository: SpikeInterface/spikeinterface
Pull request #1052 (2022-11-03 13:48:08)

Fixed crash with empty sorting
Fixed a crash where `sorting.get_all_spike_trains()` or `sorting.to_spike_vector()` would result in a crash for empty sorting objects.


Patch:
diff --git a/spikeinterface/core/basesorting.py b/spikeinterface/core/basesorting.py
index 078469fed..0f65e5a55 100644
--- a/spikeinterface/core/basesorting.py
+++ b/spikeinterface/core/basesorting.py
@@ -235,11 +235,17 @@ class BaseSorting(BaseExtractor):
                     spike_labels.append(np.array([unit_id] * st.size))
                 elif outputs == 'unit_index':
                     spike_labels.append(np.zeros(st.size, dtype='int64') + i)
-            spike_times = np.concatenate(spike_times)
-            spike_labels = np.concatenate(spike_labels)
-            order = np.argsort(spike_times)
-            spike_times = spike_times[order]
-            spike_labels = spike_labels[order]
+
+            if len(spike_times) > 0:
+                spike_times = np.concatenate(spike_times)
+                spike_labels = np.concatenate(spike_labels)
+                order = np.argsort(spike_times)
+                spike_times = spike_times[order]
+                spike_labels = spike_labels[order]
+            else:
+                spike_times = np.array([], dtype=np.int64)
+                spike_labels = np.array([], dtype=np.int64)
+
             spikes.append((spike_times, spike_labels))
         return spikes
 
diff --git a/spikeinterface/core/waveform_extractor.py b/spikeinterface/core/waveform_extractor.py
index 07728896d..bd3723b7a 100644
--- a/spikeinterface/core/waveform_extractor.py
+++ b/spikeinterface/core/waveform_extractor.py
@@ -32,9 +32,6 @@ class WaveformExtractor:
     rec_attributes: None or dict
         When recording is None then a minimal dict with some attributes
         is needed.
-    allow_unfiltered: bool
-        If true, will accept unfiltered recording.
-        False by default.
     Returns
     -------
     we: WaveformExtractor
@@ -59,7 +56,7 @@ class WaveformExtractor:
 
     """
     extensions = []
-    def __init__(self, recording, sorting, folder=None, rec_attributes=None, allow_unfiltered=False):
+    def __init__(self, recording, sorting, folder=None, rec_attributes=None):
         if recording is None:
             # this is for the mode when recording is not accessible anymore
             if rec_attributes is None:
@@ -74,11 +71,10 @@ class WaveformExtractor:
                 "The recording and sorting objects must have the same number of segments!"
             np.testing.assert_almost_equal(recording.get_sampling_frequency(),
                                            sorting.get_sampling_frequency(), decimal=2)
-            if not recording.is_filtered() and not allow_unfiltered:
+            if not recording.is_filtered():
                 raise Exception('The recording is not filtered, you must filter it using `bandpass_filter()`.'
                                 'If the recording is already filtered, you can also do '
-                                '`recording.annotate(is_filtered=True).\n'
-                                'If you trully want to extract unfiltered waveforms, use `allow_unfiltered=True`.')
+                                '`recording.annotate(is_filtered=True)')
 
         self._recording = recording
         self.sorting = sorting
@@ -153,7 +149,7 @@ class WaveformExtractor:
 
     @classmethod
     def create(cls, recording, sorting, folder, mode="folder", remove_if_exists=False,
-               use_relative_path=False, allow_unfiltered=False):
+               use_relative_path=False):
         assert mode in ("folder", "memory")
         if mode == "folder":
             folder = Path(folder)
@@ -193,7 +189,7 @@ class WaveformExtractor:
             with open(rec_attributes_file, 'r') as f:
                 rec_attributes = json.load(f)
 
-        return cls(recording, sorting, folder, allow_unfiltered=allow_unfiltered)
+        return cls(recording, sorting, folder)
 
     @classmethod
     def register_extension(cls, extension_class):
@@ -915,7 +911,6 @@ def extract_waveforms(recording, sorting, folder=None,
                       overwrite=False,
                       return_scaled=True,
                       dtype=None,
-                      allow_unfiltered=False,
                       use_relative_path=False,
                       seed=None,
                       **job_kwargs):
@@ -953,9 +948,6 @@ def extract_waveforms(recording, sorting, folder=None,
         If True and recording has gain_to_uV/offset_to_uV properties, waveforms are converted to uV.
     dtype: dtype or None
         Dtype of the output waveforms. If None, the recording dtype is maintained.
-    allow_unfiltered: bool
-        If true, will accept an allow_unfiltered recording.
-        False by default.
     use_relative_path: bool
         If True, the recording and sorting paths are relative to the waveforms folder. 
         This allows portability of the waveform folder provided that the relative paths are the same, 
@@ -981,7 +973,7 @@ def extract_waveforms(recording, sorting, folder=None,
         if load_if_exists and folder.is_dir():
             we = WaveformExtractor.load_from_folder(folder)
             return we
-    we = WaveformExtractor.create(recording, sorting, folder, mode=mode, use_relative_path=use_relative_path, allow_unfiltered=allow_unfiltered)
+    we = WaveformExtractor.create(recording, sorting, folder, mode=mode, use_relative_path=use_relative_path)
     we.set_params(ms_before=ms_before, ms_after=ms_after, max_spikes_per_unit=max_spikes_per_unit, dtype=dtype,
                   return_scaled=return_scaled)
     we.run_extract_waveforms(seed=seed, **job_kwargs)
diff --git a/spikeinterface/exporters/to_phy.py b/spikeinterface/exporters/to_phy.py
index 8b5c62bf3..32bd1e332 100644
--- a/spikeinterface/exporters/to_phy.py
+++ b/spikeinterface/exporters/to_phy.py
@@ -79,7 +79,7 @@ def export_to_phy(waveform_extractor, output_folder, compute_pc_features=True,
 
     empty_flag = False
     non_empty_units = []
-    for unit in sorting.unit_ids:
+    for unit in sorting.get_unit_ids():
         if len(sorting.get_unit_spike_train(unit)) > 0:
             non_empty_units.append(unit)
         else:
@@ -202,11 +202,12 @@ def export_to_phy(waveform_extractor, output_folder, compute_pc_features=True,
         best_channels_index = get_template_channel_sparsity(waveform_extractor, method='best_channels',
                                                             peak_sign=peak_sign, num_channels=max_channels_per_template,
                                                             outputs='index')
-        for u, unit_id in enumerate(unit_ids):
+        for u, unit_id in enumerate(sorting.unit_ids):
             pc_feature_ind[u, :] = best_channels_index[unit_id]
         np.save(str(output_folder / 'pc_feature_ind.npy'), pc_feature_ind)
 
     # Save .tsv metadata
+    unit_ids = sorting.unit_ids
     cluster_group = pd.DataFrame({'cluster_id': [i for i in range(len(unit_ids))],
                                   'group': ['unsorted'] * len(unit_ids)})
     cluster_group.to_csv(output_folder / 'cluster_group.tsv',
diff --git a/spikeinterface/sorters/si_based_sorters/spyking_circus2.py b/spikeinterface/sorters/si_based_sorters/spyking_circus2.py
index e5d9bcf93..c607a67c5 100644
--- a/spikeinterface/sorters/si_based_sorters/spyking_circus2.py
+++ b/spikeinterface/sorters/si_based_sorters/spyking_circus2.py
@@ -105,9 +105,7 @@ class Spykingcircus2Sorter(ComponentsBasedSorter):
         clustering_params = params['clustering'].copy()
         clustering_params.update(params['waveforms'])
         clustering_params.update(params['general'])
-        clustering_params.update(dict(shared_memory=params['shared_memory']))
         clustering_params['job_kwargs'] = params['job_kwargs']
-        clustering_params['tmp_folder'] = output_folder / "clustering"
 
         labels, peak_labels = find_cluster_from_peaks(recording_f, selected_peaks, method='random_projections',
             method_kwargs=clustering_params)
diff --git a/spikeinterface/sortingcomponents/clustering/random_projections.py b/spikeinterface/sortingcomponents/clustering/random_projections.py
index 46a81f7d4..f60a74072 100644
--- a/spikeinterface/sortingcomponents/clustering/random_projections.py
+++ b/spikeinterface/sortingcomponents/clustering/random_projections.py
@@ -36,7 +36,6 @@ class RandomProjectionClustering:
         "cleaning_method": "matching",
         "shared_memory" : False,
         "min_values" : {'ptp' : 0, 'energy' : 0},
-        "tmp_folder" : None,
         "job_kwargs" : {"n_jobs" : os.cpu_count(), "chunk_memory" : "10M", "verbose" : True, "progress_bar" : True},
     }
 
@@ -173,23 +172,20 @@ class RandomProjectionClustering:
             labels, peak_labels = remove_duplicates_via_dip(wfs_arrays, peak_labels, **params['cleaning_kwargs'])
 
         elif cleaning_method == "matching":
-            # create a tmp folder
-            if params["tmp_folder"] is None:
-                name = ''.join(random.choices(string.ascii_uppercase + string.digits, k=8))
-                tmp_folder = get_global_tmp_folder() / name
-            else:
-                tmp_folder = params["tmp_folder"]
+
             if params['shared_memory']:
-                waveform_folder = None
+                tmp_folder = None
                 mode = 'memory'
             else:
-                waveform_folder = tmp_folder / "waveforms"
+                name = ''.join(random.choices(string.ascii_uppercase + string.digits, k=8))
+                tmp_folder = Path(os.path.join(get_global_tmp_folder(), name))
+                if tmp_folder.exists():
+                    import shutils
+                    shutils.rmtree(tmp_folder)
                 mode = 'folder'
 
-            sorting_folder = tmp_folder / "sorting"
             sorting = NumpySorting.from_times_labels(spikes['sample_ind'], spikes['unit_ind'], fs)
-            sorting = sorting.save(folder=sorting_folder)
-            we = extract_waveforms(recording, sorting, waveform_folder, ms_before=params['ms_before'],
+            we = extract_waveforms(recording, sorting, tmp_folder, ms_before=params['ms_before'], 
                 ms_after=params['ms_after'], **params['job_kwargs'], return_scaled=False, mode=mode)
 
             cleaning_matching_params = params['job_kwargs'].copy()
@@ -198,14 +194,10 @@ class RandomProjectionClustering:
             cleaning_matching_params['verbose'] = False
             cleaning_matching_params['progress_bar'] = False
 
-            labels, peak_labels = remove_duplicates_via_matching(we, noise_levels, peak_labels, 
-                                                                 job_kwargs=cleaning_matching_params, **params['cleaning_kwargs'])
+            labels, peak_labels = remove_duplicates_via_matching(we, noise_levels, peak_labels, job_kwargs=cleaning_matching_params, **params['cleaning_kwargs'])
     
-            if params["tmp_folder"] is None:
+            if tmp_folder is not None:
                 shutil.rmtree(tmp_folder)
-            else:
-                shutil.rmtree(tmp_folder / "waveforms")
-                shutil.rmtree(tmp_folder / "sorting")
 
         if verbose:
             print("We kept %d non-duplicated clusters..." %len(labels))


Test patch:
diff --git a/spikeinterface/core/tests/test_basesorting.py b/spikeinterface/core/tests/test_basesorting.py
index 824f8e0f9..64dbeb990 100644
--- a/spikeinterface/core/tests/test_basesorting.py
+++ b/spikeinterface/core/tests/test_basesorting.py
@@ -7,7 +7,7 @@ from pathlib import Path
 import pytest
 import numpy as np
 
-from spikeinterface.core import NpzSortingExtractor, load_extractor
+from spikeinterface.core import NumpySorting, NpzSortingExtractor, load_extractor
 from spikeinterface.core.base import BaseExtractor
 
 from spikeinterface.core.testing_tools import create_sorting_npz, generate_sorting
@@ -89,5 +89,20 @@ def test_BaseSorting():
         assert unit not in empty_units
 
 
+def test_empty_sorting():
+    sorting = NumpySorting.from_dict({}, 30000)
+
+    assert len(sorting.unit_ids) == 0
+
+    spikes = sorting.get_all_spike_trains()
+    assert len(spikes) == 1
+    assert len(spikes[0][0]) == 0
+    assert len(spikes[0][1]) == 0
+
+    spikes = sorting.to_spike_vector()
+    assert spikes.shape == (0, )
+
+
 if __name__ == '__main__':
     test_BaseSorting()
+    test_empty_sorting()
diff --git a/spikeinterface/core/tests/test_waveform_extractor.py b/spikeinterface/core/tests/test_waveform_extractor.py
index 290bfe232..758298038 100644
--- a/spikeinterface/core/tests/test_waveform_extractor.py
+++ b/spikeinterface/core/tests/test_waveform_extractor.py
@@ -142,18 +142,6 @@ def test_extract_waveforms():
     wf_mem = we_mem.get_waveforms(0)
     assert np.array_equal(wf_mem, wf3)
 
-    # Test unfiltered recording
-    recording.annotate(is_filtered=False)
-
-    folder_crash = cache_folder / "test_extract_waveforms_crash"
-    with pytest.raises(Exception):
-        we1 = extract_waveforms(recording, sorting, folder_crash,
-                                max_spikes_per_unit=None, return_scaled=False)
-
-    folder_unfiltered = cache_folder / "test_extract_waveforms_unfiltered"
-    we1 = extract_waveforms(recording, sorting, folder_unfiltered, allow_unfiltered=True,
-                            max_spikes_per_unit=None, return_scaled=False)
-
 
 def test_recordingless():
     durations = [30, 40]