Repository: SpikeInterface/spikeinterface
Pull request #483 (2022-01-28 08:53:25)

Add 'missing_value' to set_property
This PR allows for better control of missing values when setting properties by adding the argument `missing_value`

Note that in case the missing value and the values have different dtypes, the property dtype is set to `object`


Patch:
diff --git a/spikeinterface/core/base.py b/spikeinterface/core/base.py
index 80e6ab5a7..576f87216 100644
--- a/spikeinterface/core/base.py
+++ b/spikeinterface/core/base.py
@@ -4,9 +4,9 @@ from copy import deepcopy
 import weakref
 import json
 import pickle
-import datetime
 import random
 import string
+import warnings
 
 import numpy as np
 
@@ -131,16 +131,31 @@ class BaseExtractor:
     def get_annotation_keys(self):
         return list(self._annotations.keys())
 
-    def set_property(self, key, values, ids=None):
+    def set_property(self, key, values, ids=None, missing_value=None):
         """
-        Set property vector:
-          * channel_property
-          * unit_property
+        Set property vector for main ids.
 
         If ids is given AND property already exists,
-        then it is modified only on a subset of channels/units
+        then it is modified only on a subset of channels/units.
+        missing_values allows to specify the values of unset 
+        properties if ids is used
 
+
+        Parameters
+        ----------
+        key : str
+            The property name
+        values : np.array
+            Array of values for the property
+        ids : list/np.array, optional
+            List of subset of ids to set the values, by default None
+        missing_value : object, optional
+            In case the property is set on a subset of values ('ids' not None), 
+            it specifies the how the missing values should be filled, by default None.
+            The missing_value has to be specified for types int and unsigned int.
         """
+        default_missing_values = {"f": np.nan, "S": "", "U": ""}
+        
         if values is None:
             if key in self._properties:
                 self._properties.pop(key)
@@ -148,26 +163,48 @@ class BaseExtractor:
 
         size = self._main_ids.size
         values = np.asarray(values)
+        dtype = values.dtype
+        dtype_kind = dtype.kind
+        
         if ids is None:
             assert values.shape[0] == size
             self._properties[key] = values
         else:
-            if key not in self._properties:
-                # create the property with nan or empty
-                shape = (size,) + values.shape[1:]
-                if values.dtype.kind in ('i', 'f', 'S', 'U'):
-                    dtype = values.dtype
+            ids = np.array(ids)
+            assert np.unique(ids).size == ids.size, "'ids' are not unique!"
+            
+            if ids.size < size:
+                if key not in self._properties:
+                    # create the property with nan or empty
+                    shape = (size,) + values.shape[1:]
+                    
+        
+                    if missing_value is None:
+                        if dtype_kind not in default_missing_values.keys():
+                            raise Exception("For values dtypes other than float, string or unicode, the missing value "
+                                            "cannot be automatically inferred. Please specify it with the 'missing_value' "
+                                            "argument.")
+                        else:
+                            missing_value = default_missing_values[dtype_kind]
+                    else:
+                        assert dtype_kind == np.array(missing_value).dtype.kind, ("Mismatch between values and "
+                                                                                  "missing_value types. Provide a "
+                                                                                  "missing_value with the same type "
+                                                                                  "as the values.")
+                        
+                    empty_values = np.zeros(shape, dtype=dtype)
+                    empty_values[:] = missing_value
+                    self._properties[key] = empty_values
                 else:
-                    dtype = object
-                empty_values = np.zeros(shape, dtype=dtype)
-                if values.dtype.kind == 'f':
-                    empty_values = empty_values * np.nan
-                # ~ elif values.dtype.kind == 'i':
-                # ~ # TODO find a way to put missing values
-                self._properties[key] = empty_values
-
-            indices = self.ids_to_indices(ids)
-            self._properties[key][indices] = values
+                    assert dtype_kind == self._properties[key].dtype.kind, ("Mismatch between existing property dtype "
+                                                                            "values dtype.")
+
+                indices = self.ids_to_indices(ids)
+                self._properties[key][indices] = values
+            else:
+                indices = self.ids_to_indices(ids)
+                self._properties[key] = np.zeros_like(values, dtype=values.dtype)
+                self._properties[key][indices] = values
 
     def get_property(self, key, ids=None):
         values = self._properties.get(key, None)
diff --git a/spikeinterface/core/old_api_utils.py b/spikeinterface/core/old_api_utils.py
index 5874dc851..bfeafbebd 100644
--- a/spikeinterface/core/old_api_utils.py
+++ b/spikeinterface/core/old_api_utils.py
@@ -1,6 +1,7 @@
 from typing import Union
 
 import numpy as np
+import warnings
 from spikeinterface.core import (BaseRecording, BaseSorting,
                                  BaseRecordingSegment, BaseSortingSegment)
 
@@ -322,10 +323,23 @@ def copy_properties(oldapi_extractor, new_extractor, skip_properties=None):
             properties[prop_name]["values"].append(prop_value)
 
     for property_name, prop_dict in properties.items():
-        property_ids = prop_dict["ids"]
-        property_values = prop_dict["values"]
-        new_extractor.set_property(key=property_name,
-                                   values=property_values, ids=property_ids)
+        property_ids = np.array(prop_dict["ids"])
+        property_values = np.array(prop_dict["values"])
+        missing_value = None
+        
+        # For back-compatibility, incomplete int/uint properties are upcast to float
+        # and missing_value is set to np.nan
+        if len(property_ids) < len(get_ids()):
+            if property_values.dtype.kind in ("u", "i"):
+                property_values = property_values.astype("float")
+                missing_value = np.nan
+        try:
+            new_extractor.set_property(key=property_name,
+                                       values=property_values, 
+                                       ids=property_ids,
+                                       missing_value=missing_value)
+        except Exception as e:
+            warnings.warn(f"Property {property_name} cannot be ported to new API due to missing values.")
 
 
 def find_old_gains_offsets_recursively(oldapi_extractor_dict):


Test patch:
diff --git a/spikeinterface/core/tests/test_baserecording.py b/spikeinterface/core/tests/test_baserecording.py
index be769efe0..f67b35f9a 100644
--- a/spikeinterface/core/tests/test_baserecording.py
+++ b/spikeinterface/core/tests/test_baserecording.py
@@ -6,6 +6,7 @@ import shutil
 from pathlib import Path
 import pytest
 import numpy as np
+from numpy.testing import assert_raises
 
 from probeinterface import Probe
 
@@ -59,6 +60,22 @@ def test_BaseRecording():
     values = rec.get_property('quality')
     assert np.all(values[:2] == [1., 3.3, ])
     
+    # missing property
+    rec.set_property('string_property', ["ciao", "bello"], ids=[0, 1])
+    values = rec.get_property('string_property')
+    assert values[2] == ""
+    
+    # setting an different type raises an error
+    assert_raises(Exception, rec.set_property, key='string_property_nan', values=["ciao", "bello"], ids=[0, 1], 
+                  missing_value=np.nan)
+    
+    # int properties without missing values raise an error
+    assert_raises(Exception, rec.set_property, key='int_property', values=[5, 6], ids=[1, 2])
+    
+    rec.set_property('int_property', [5, 6], ids=[1, 2], missing_value=200)
+    values = rec.get_property('int_property')
+    assert values.dtype.kind == "i"
+    
     times0 = rec.get_times(segment_index=0)
 
     # dump/load dict