Repository: SpikeInterface/spikeinterface
Pull request #114 (2021-04-22 16:44:20)

update readthedoc requirements


Patch:
diff --git a/examples/modules/extractors/plot_4_working_with_unscaled_traces.py b/examples/modules/extractors/plot_4_working_with_unscaled_traces.py
index a3dd1dd7a..0e6fb7886 100644
--- a/examples/modules/extractors/plot_4_working_with_unscaled_traces.py
+++ b/examples/modules/extractors/plot_4_working_with_unscaled_traces.py
@@ -10,53 +10,62 @@ module.
 
 #TODO
 
-#~ import numpy as np
-#~ import matplotlib.pyplot as plt
-#~ import spikeinterface.extractors as se
+import numpy as np
+import matplotlib.pyplot as plt
+import spikeinterface.extractors as se
 
-#~ ##############################################################################
-#~ # First, let's create some traces in unsigned int16 type. Assuming the ADC output of our recording system has 10 bits,
-#~ # the values will be between 0 and 1024. Let's assume our signal is centered at 512 and it has a standard deviation
-#~ # of 50 bits
+##############################################################################
+# First, let's create some traces in unsigned int16 type. Assuming the ADC output of our recording system has 10 bits,
+# the values will be between 0 and 1024. Let's assume our signal is centered at 512 and it has a standard deviation
+# of 50 bits
 
-#~ sampling_frequency = 30000
-#~ traces = 512 + 50 * np.random.randn(4, 10*sampling_frequency)
-#~ traces = traces.astype("uint16")
+sampling_frequency = 30000
+traces = 512 + 50 * np.random.randn(10*sampling_frequency, 4)
+traces = traces.astype("uint16")
 
-#~ ###############################################################################
-#~ # Let's now instantiate a :code:`NumpyRecordingExtractor` with the traces we just created
+###############################################################################
+# Let's now instantiate a :code:`NumpyRecordingExtractor` with the traces we just created
 
-#~ recording = se.NumpyRecordingExtractor(traces, sampling_frequency=sampling_frequency)
-#~ print(f"Traces dtype: {recording.get_dtype()}")
+recording = se.NumpyRecording([traces], sampling_frequency=sampling_frequency)
+print(f"Traces dtype: {recording.get_dtype()}")
 
-#~ ###############################################################################
-#~ # Since our ADC samples between 0 and 1024, we need to convert to uV. To do so, we need to transform the traces as:
-#~ # traces_uV = traces_raw * gains + offset
-#~ #
-#~ # Let's assume that our gain (i.e. the value of each bit) is 0.1, so that our voltage range is between 0 and 1024*0.1.
-#~ # We also need an offset to center the traces around 0. The offset will be:  - 2^(10-1) * gain = -512 * gain
-#~ # (where 10 is the number of bits of our ADC)
+###############################################################################
+# Since our ADC samples between 0 and 1024, we need to convert to uV. To do so, we need to transform the traces as:
+# traces_uV = traces_raw * gains + offset
+#
+# Let's assume that our gain (i.e. the value of each bit) is 0.1, so that our voltage range is between 0 and 1024*0.1.
+# We also need an offset to center the traces around 0. The offset will be:  - 2^(10-1) * gain = -512 * gain
+# (where 10 is the number of bits of our ADC)
 
-#~ gain = 0.1
-#~ offset = -2**(10 - 1) * gain
+gain = 0.1
+offset = -2**(10 - 1) * gain
 
-#~ ###############################################################################
-#~ # We are now ready to set gains and offsets to our extractor. We also have to set the :code:`has_unscaled` field to
-#~ # :code:`True`:
+###############################################################################
+# We are now ready to set gains and offsets to our extractor. We also have to set the :code:`has_unscaled` field to
+# :code:`True`:
 
-#~ recording.set_channel_gains(gain)
-#~ recording.set_channel_offsets(offset)
-#~ recording.has_unscaled = True
+recording.set_channel_gains(gain)
+recording.set_channel_offsets(offset)
 
-#~ ###############################################################################
-#~ # With gains and offset information, we can retrieve traces both in their unscaled (raw) type, and in their scaled
-#~ # type:
+###############################################################################
+# Internally this gains and offsets are handle with properties
+# So the gain could be "by channel".
 
-#~ traces_unscaled = recording.get_traces(return_scaled=False)
-#~ traces_scaled = recording.get_traces(return_scaled=True)  # return_scaled is True by default
+print(recording.get_property('gain_to_uV'))
+print(recording.get_property('offset_to_uV'))
 
-#~ print(f"Traces dtype after scaling: {recording.get_dtype(return_scaled=True)}")
 
-#~ plt.plot(traces_unscaled[0], label="unscaled")
-#~ plt.plot(traces_scaled[0], label="scaled")
-#~ plt.legend()
+###############################################################################
+# With gains and offset information, we can retrieve traces both in their unscaled (raw) type, and in their scaled
+# type:
+
+traces_unscaled = recording.get_traces(return_scaled=False)
+traces_scaled = recording.get_traces(return_scaled=True)  # return_scaled is True by default
+
+print(f"Traces dtype after scaling: {traces_scaled.dtype}")
+
+plt.plot(traces_unscaled[:, 0], label="unscaled")
+plt.plot(traces_scaled[:, 0], label="scaled")
+plt.legend()
+
+plt.show()
diff --git a/readthedocs.yml b/readthedocs.yml
index ed17a5efa..b39e35e86 100644
--- a/readthedocs.yml
+++ b/readthedocs.yml
@@ -2,8 +2,6 @@ build:
     image: latest
 
 python:
-  version: 3.6
+  version: 3.8
   install:
-    - requirements: requirements.txt
-    - requirements: requirements_test.txt
     - requirements: requirements_rtd.txt
diff --git a/requirements_rtd.txt b/requirements_rtd.txt
index 0c97fe480..d5afdca51 100644
--- a/requirements_rtd.txt
+++ b/requirements_rtd.txt
@@ -1,15 +1,27 @@
-# optional dependancy
-h5py
-pandas
-sklearn
+# main dependancy
+numpy
+neo
+joblib
+tqdm
+probeinterface
 
 # sphinx-gallery
 pillow
 sphinx-gallery
 numpydoc
-matplotlib==3.2.2
+matplotlib
 
 
+# optional dependancy for sub modules
+h5py
+pandas
+scipy
+sklearn
+matplotlib
+networkx
+datalad
+MEArec
+
 ## install sorters ##
 # ms4
 mountainsort4==1.0.0
@@ -24,4 +36,4 @@ klustakwik2==0.2.7
 # tridesclous
 pyqt5
 hdbscan
-tridesclous==1.6.0
+tridesclous
diff --git a/spikeinterface/core/baserecording.py b/spikeinterface/core/baserecording.py
index ccd031040..80726a1ac 100644
--- a/spikeinterface/core/baserecording.py
+++ b/spikeinterface/core/baserecording.py
@@ -81,6 +81,7 @@ class BaseRecording(BaseExtractor):
                    end_frame: Union[SampleIndex, None] = None,
                    channel_ids: Union[List[ChannelId], None] = None,
                    order: Union[Order, None] = None,
+                   return_scaled=False,
                    ):
         segment_index = self._check_segment_index(segment_index)
         channel_indices = self.ids_to_indices(channel_ids, prefer_slice=True)
@@ -88,6 +89,14 @@ class BaseRecording(BaseExtractor):
         traces = rs.get_traces(start_frame=start_frame, end_frame=end_frame, channel_indices=channel_indices)
         if order is not None:
             traces = np.asanyarray(traces, order=order)
+        if return_scaled:
+            gains = self.get_property('gain_to_uV')
+            offsets = self.get_property('offset_to_uV')
+            if gains is None or offsets is None:
+                raise ValueError('This recording do not support return_scaled=True (need gain_to_uV and offset_to_uV properties)')
+            gains = gains[channel_indices].astype('float32')
+            offsets = offsets[channel_indices].astype('float32')
+            traces = traces.astype('float32') * gains + offsets
         return traces
 
     def is_filtered(self):
@@ -351,10 +360,20 @@ class BaseRecording(BaseExtractor):
         self.set_property('group', groups, ids=channel_ids)
 
     def set_channel_gains(self, gains, channel_ids=None):
-        self.set_property('gain', gains, ids=channel_ids)
+        if np.isscalar(gains):
+            gains = [gains] * self.get_num_channels()
+        self.set_property('gain_to_uV', gains, ids=channel_ids)
 
     def get_channel_gains(self, channel_ids=None):
-        return self.get_property('gain', ids=channel_ids)
+        return self.get_property('gain_to_uV', ids=channel_ids)
+
+    def set_channel_offsets(self, offsets, channel_ids=None):
+        if np.isscalar(offsets):
+            offsets = [offsets] * self.get_num_channels()
+        self.set_property('offset_to_uV', offsets, ids=channel_ids)
+
+    def get_channel_offsets(self, channel_ids=None):
+        return self.get_property('offset_to_uV', ids=channel_ids)
 
     def get_channel_property(self, channel_id, key):
         values = self.get_property(key)
diff --git a/spikeinterface/extractors/neoextractors/neobaseextractor.py b/spikeinterface/extractors/neoextractors/neobaseextractor.py
index c31c97329..8b0801add 100644
--- a/spikeinterface/extractors/neoextractors/neobaseextractor.py
+++ b/spikeinterface/extractors/neoextractors/neobaseextractor.py
@@ -52,35 +52,42 @@ class NeoBaseRecordingExtractor(_NeoBaseExtractor, BaseRecording):
         raw_dtypes = signal_channels['dtype']
 
         sampling_frequency = self.neo_reader.get_signal_sampling_rate(stream_index=self.stream_index)
-        # TODO propose a mechanism to select scaled/raw dtype
-        scaled_dtype = 'float32'
-        BaseRecording.__init__(self, sampling_frequency, chan_ids, scaled_dtype)
+        dtype = signal_channels['dtype'][0]
+        BaseRecording.__init__(self, sampling_frequency, chan_ids, dtype)
+
+        # find the gain to uV
+        gains = signal_channels['gain']
+        offsets = signal_channels['offset']
 
-        # spikeinterface for units to be uV implicitly
         units = signal_channels['units']
         if not np.all(np.isin(units, ['V', 'mV', 'uV'])):
             # check that units are V, mV or uV
             error = f'This extractor based on  neo.{self.NeoRawIOClass} have strange units not in (V, mV, uV) {units}'
             print(error)
-        self.additional_gain = np.ones(units.size, dtype='float')
-        self.additional_gain[units == 'V'] = 1e6
-        self.additional_gain[units == 'mV'] = 1e3
-        self.additional_gain[units == 'uV'] = 1.
-        self.additional_gain = self.additional_gain.reshape(1, -1)
-
+        additional_gain = np.ones(units.size, dtype='float')
+        additional_gain[units == 'V'] = 1e6
+        additional_gain[units == 'mV'] = 1e3
+        additional_gain[units == 'uV'] = 1.
+        additional_gain = additional_gain
+
+        final_gains = gains * additional_gain
+        final_offsets = offsets * additional_gain
+        
+        self.set_property('gain_to_uV', final_gains)
+        self.set_property('offset_to_uV', final_offsets)
+        
         nseg = self.neo_reader.segment_count(block_index=0)
         for segment_index in range(nseg):
-            rec_segment = NeoRecordingSegment(self.neo_reader, segment_index, self.stream_index, self.additional_gain)
+            rec_segment = NeoRecordingSegment(self.neo_reader, segment_index, self.stream_index)
             self.add_recording_segment(rec_segment)
 
 
 class NeoRecordingSegment(BaseRecordingSegment):
-    def __init__(self, neo_reader, segment_index, stream_index, additional_gain):
+    def __init__(self, neo_reader, segment_index, stream_index):
         BaseRecordingSegment.__init__(self)
         self.neo_reader = neo_reader
         self.segment_index = segment_index
         self.stream_index = stream_index
-        self.additional_gain = additional_gain
 
     def get_num_samples(self):
         n = self.neo_reader.get_signal_size(block_index=0,
@@ -88,9 +95,7 @@ class NeoRecordingSegment(BaseRecordingSegment):
                                             stream_index=self.stream_index)
         return n
 
-    def get_traces(self, start_frame, end_frame, channel_indices, return_scaled=True):
-        # in neo rawio channel can acces by names/ids/indexes
-        # there is no garranty that ids/names are unique on some formats
+    def get_traces(self, start_frame, end_frame, channel_indices):
         raw_traces = self.neo_reader.get_analogsignal_chunk(
             block_index=0,
             seg_index=self.segment_index,
@@ -99,19 +104,7 @@ class NeoRecordingSegment(BaseRecordingSegment):
             stream_index=self.stream_index,
             channel_indexes=channel_indices,
         )
-
-        if return_scaled:
-            # rescale traces to natural units (can be anything)
-            scaled_traces = self.neo_reader.rescale_signal_raw_to_float(raw_traces,
-                                                                        dtype='float32', stream_index=self.stream_index,
-                                                                        channel_indexes=channel_indices)
-            # and then to uV
-            if channel_indices is None:
-                channel_indices = slice(None)
-            scaled_traces *= self.additional_gain[:, channel_indices]
-            return scaled_traces
-        else:
-            return raw_traces
+        return raw_traces
 
 
 class NeoBaseSortingExtractor(_NeoBaseExtractor, BaseSorting):


Test patch:
diff --git a/.github/workflows/full-test.yml b/.github/workflows/full-test.yml
index 552150c42..110fd4982 100644
--- a/.github/workflows/full-test.yml
+++ b/.github/workflows/full-test.yml
@@ -2,7 +2,7 @@ name: Full spikeinterface tests
 
 on:
   pull_request:
-    branches: [master, big_refactoring]
+    branches: [master]
     types: [synchronize, opened, reopened, ready_for_review]
 
 jobs:
@@ -24,10 +24,10 @@ jobs:
       - uses: actions/cache@v2
         id: cache-venv
         with:
-          path: ./test_env
-          key: $venv-{{ runner.os }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements_test.txt') }}
+          path: ~/test_env
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements_test.txt') }}
           restore-keys: |
-            ${{ runner.os }}-venv-
+            ${{ runner.os }}-venv
       - name: Install dependencies
         run: |
           # this is for datalad and download testing datasets
@@ -38,8 +38,8 @@ jobs:
           # this is for spyking circus
           sudo apt install mpich libmpich-dev
           # create an environement (better for cahing)
-          python -m venv test_env
-          source ./test_env/bin/activate
+          python -m venv ~/test_env
+          source ~/test_env/bin/activate
           python -m pip install --upgrade pip
           pip install setuptools wheel twine
           ##
@@ -50,6 +50,11 @@ jobs:
           pip install -r requirements.txt
           pip install -r requirements_test.txt
           pip install -e .
+      - name: Get ephy_testing_data current head hash
+        # the key depend on the last comit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
+        id: vars
+        run: |
+          echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)"
       - uses: actions/cache@v2
         id: cache-datasets
         env:
@@ -57,12 +62,12 @@ jobs:
           HASH_EPHY_DATASET: git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1
         with:
           path: ~/spikeinterface_datasets
-          key: ${{ runner.os }}-datasets-${HASH_EPHY_DATASET}
+          key: ${{ runner.os }}-datasets-${{ steps.vars.outputs.HASH_EPHY_DATASET }}
           restore-keys: |
             ${{ runner.os }}-datasets
       - name: Test with pytest
         run: |
-          source ./test_env/bin/activate
+          source ~/test_env/bin/activate
           pytest
 
           
diff --git a/requirements_test.txt b/requirements_test.txt
index 773a6feb0..91b87a866 100644
--- a/requirements_test.txt
+++ b/requirements_test.txt
@@ -3,6 +3,7 @@ pytest
 # optional dependancy for sub modules
 h5py
 pandas
+scipy
 sklearn
 matplotlib
 networkx
diff --git a/spikeinterface/core/tests/test_baserecording.py b/spikeinterface/core/tests/test_baserecording.py
index f3da1640d..10eef2f57 100644
--- a/spikeinterface/core/tests/test_baserecording.py
+++ b/spikeinterface/core/tests/test_baserecording.py
@@ -10,7 +10,7 @@ import numpy as np
 
 from probeinterface import Probe
 
-from spikeinterface.core import BinaryRecordingExtractor, load_extractor
+from spikeinterface.core import BinaryRecordingExtractor, NumpyRecording, load_extractor
 from spikeinterface.core.base import BaseExtractor
 
 # file and folder created
@@ -112,8 +112,24 @@ def test_BaseRecording():
     # import matplotlib.pyplot as plt
     # plot_probe(probe)
     # plot_probe(probe2)
-    # plt.show()    
-
+    # plt.show()
+    
+    # test return_scale
+    sampling_frequency = 30000
+    traces = np.zeros((1000, 5), dtype='int16')
+    rec_int16 = NumpyRecording([traces], sampling_frequency)
+    assert rec_int16.get_dtype() == 'int16'
+    print(rec_int16)
+    traces_int16 = rec_int16.get_traces()
+    assert traces_int16.dtype == 'int16'
+    # return_scaled raise error when no gain_to_uV/offset_to_uV properties
+    with pytest.raises(ValueError):
+        traces_float32 = rec_int16.get_traces(return_scaled=True)
+    rec_int16.set_property('gain_to_uV', [.195]*5)
+    rec_int16.set_property('offset_to_uV', [0.]*5)
+    traces_float32 = rec_int16.get_traces(return_scaled=True)
+    assert traces_float32.dtype == 'float32'
+    
 
 if __name__ == '__main__':
     _clean_all()
diff --git a/spikeinterface/extractors/tests/common_tests.py b/spikeinterface/extractors/tests/common_tests.py
index 88e6c4e72..0866e98dd 100644
--- a/spikeinterface/extractors/tests/common_tests.py
+++ b/spikeinterface/extractors/tests/common_tests.py
@@ -3,6 +3,7 @@ import pytest
 import numpy as np
 
 from spikeinterface import download_dataset, get_global_dataset_folder
+from spikeinterface.extractors.neoextractors.neobaseextractor import NeoBaseRecordingExtractor
 
 
 gin_repo = 'https://gin.g-node.org/NeuralEnsemble/ephy_testing_data'
@@ -43,6 +44,7 @@ class RecordingCommonTestSuite(CommonTestSuite):
                 assert full_traces.shape == (num_samples, num_chans)
                 assert full_traces.dtype == dtype
                 
+                
                 traces_sample_first = rec.get_traces(segment_index=segment_index, start_frame=0, end_frame=1)
                 assert traces_sample_first.shape == (1, num_chans)
                 assert np.all(full_traces[0, :] == traces_sample_first[0, :])
@@ -50,7 +52,15 @@ class RecordingCommonTestSuite(CommonTestSuite):
                 traces_sample_last = rec.get_traces(segment_index=segment_index, start_frame=num_samples-1, end_frame=num_samples)
                 assert traces_sample_last.shape == (1, num_chans)
                 assert np.all(full_traces[-1, :] == traces_sample_last[0, :])
-
+            
+            # try return_scaled
+            if isinstance(rec, NeoBaseRecordingExtractor):
+                assert rec.get_property('gain_to_uV') is not None
+                assert rec.get_property('offset_to_uV') is not None
+                
+            if rec.get_property('gain_to_uV') is not None and rec.get_property('offset_to_uV') is not None:
+                trace_scaled = rec.get_traces(segment_index=segment_index, return_scaled=True)
+                assert trace_scaled.dtype == 'float32'
 
 
 
diff --git a/spikeinterface/extractors/tests/test_neoextractors.py b/spikeinterface/extractors/tests/test_neoextractors.py
index 48dc4fc09..d833e0e70 100644
--- a/spikeinterface/extractors/tests/test_neoextractors.py
+++ b/spikeinterface/extractors/tests/test_neoextractors.py
@@ -109,10 +109,10 @@ class KiloSortSortingTest(SortingCommonTestSuite, unittest.TestCase):
 
 
 if __name__ == '__main__':
-    # test = MearecRecordingTest()
-    test = MearecSortingTest()
-    # test = SpikeGLXRecordingTest()
-    # test = OpenEphysBinaryRecordingTest()
+    #~ test = MearecRecordingTest()
+    #~ test = MearecSortingTest()
+    test = SpikeGLXRecordingTest()
+    #~ test = OpenEphysBinaryRecordingTest()
     # test = OpenEphysLegacyRecordingTest()
     # test = ItanRecordingTest()
     # test = NeuroScopeRecordingTest()