nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: SpikeInterface/spikeinterface
Pull request #2940 (2024-05-30 14:54:49)

Add limit for `n_jobs` with a max of `os.cpu_count()`
Based on discussion in #2875.

@samuelgarcia still needs to weigh in. But this is just the proposal so can be closed if we don't want this.


Patch:
diff --git a/src/spikeinterface/core/job_tools.py b/src/spikeinterface/core/job_tools.py
index 3849c2b6e..a5279247f 100644
--- a/src/spikeinterface/core/job_tools.py
+++ b/src/spikeinterface/core/job_tools.py
@@ -96,7 +96,8 @@ def fix_job_kwargs(runtime_job_kwargs):
     else:
         n_jobs = int(n_jobs)
 
-    job_kwargs["n_jobs"] = max(n_jobs, 1)
+    n_jobs = max(n_jobs, 1)
+    job_kwargs["n_jobs"] = min(n_jobs, os.cpu_count())
 
     if "n_jobs" not in runtime_job_kwargs and job_kwargs["n_jobs"] == 1 and not is_set_global_job_kwargs_set():
         warnings.warn(


Test patch:
diff --git a/src/spikeinterface/core/tests/test_globals.py b/src/spikeinterface/core/tests/test_globals.py
index a45bb6f49..668cdb980 100644
--- a/src/spikeinterface/core/tests/test_globals.py
+++ b/src/spikeinterface/core/tests/test_globals.py
@@ -1,6 +1,7 @@
 import pytest
 import warnings
 from pathlib import Path
+from os import cpu_count
 
 from spikeinterface import (
     set_global_dataset_folder,
@@ -64,7 +65,7 @@ def test_global_job_kwargs():
         n_jobs=2, chunk_duration="1s", progress_bar=True, mp_context=None, max_threads_per_process=1
     )
     # test that fix_job_kwargs grabs global kwargs
-    new_job_kwargs = dict(n_jobs=10)
+    new_job_kwargs = dict(n_jobs=cpu_count())
     job_kwargs_split = fix_job_kwargs(new_job_kwargs)
     assert job_kwargs_split["n_jobs"] == new_job_kwargs["n_jobs"]
     assert job_kwargs_split["chunk_duration"] == job_kwargs["chunk_duration"]
@@ -74,6 +75,10 @@ def test_global_job_kwargs():
     job_kwargs_split = fix_job_kwargs(none_job_kwargs)
     assert job_kwargs_split["chunk_duration"] == job_kwargs["chunk_duration"]
     assert job_kwargs_split["progress_bar"] == job_kwargs["progress_bar"]
+    # test that n_jobs are clipped if using more than virtual cores
+    excessive_n_jobs = dict(n_jobs=cpu_count() + 2)
+    job_kwargs_split = fix_job_kwargs(excessive_n_jobs)
+    assert job_kwargs_split["n_jobs"] == cpu_count()
     reset_global_job_kwargs()
 
 
diff --git a/src/spikeinterface/core/tests/test_job_tools.py b/src/spikeinterface/core/tests/test_job_tools.py
index f23a56ac5..2f3aff002 100644
--- a/src/spikeinterface/core/tests/test_job_tools.py
+++ b/src/spikeinterface/core/tests/test_job_tools.py
@@ -188,9 +188,9 @@ def test_fix_job_kwargs():
         assert fixed_job_kwargs["n_jobs"] == 1
 
     # test float value > 1 is cast to correct int
-    job_kwargs = dict(n_jobs=4.0, progress_bar=False, chunk_duration="1s")
+    job_kwargs = dict(n_jobs=float(os.cpu_count()), progress_bar=False, chunk_duration="1s")
     fixed_job_kwargs = fix_job_kwargs(job_kwargs)
-    assert fixed_job_kwargs["n_jobs"] == 4
+    assert fixed_job_kwargs["n_jobs"] == os.cpu_count()
 
     # test wrong keys
     with pytest.raises(AssertionError):
Repository: SpikeInterface/spikeinterface
Pull request #3103 (2024-06-28 21:36:33)

Accept paths in `SIJsonEncoder`. 
Sometimes recording might have annotations that or other arguments that are paths and jsonification of dictionaries will fail. This PR solves this by adding another route for the json encoder to do this and some tests.


Patch:
diff --git a/src/spikeinterface/core/core_tools.py b/src/spikeinterface/core/core_tools.py
index 4a40d8e42..1f2e644be 100644
--- a/src/spikeinterface/core/core_tools.py
+++ b/src/spikeinterface/core/core_tools.py
@@ -99,6 +99,9 @@ class SIJsonEncoder(json.JSONEncoder):
         if isinstance(obj, BaseExtractor):
             return obj.to_dict()
 
+        if isinstance(obj, Path):
+            return str(obj)
+
         if isinstance(obj, Motion):
             return obj.to_dict()
 


Test patch:
diff --git a/src/spikeinterface/core/tests/test_jsonification.py b/src/spikeinterface/core/tests/test_jsonification.py
index 4417ea342..316dac3ab 100644
--- a/src/spikeinterface/core/tests/test_jsonification.py
+++ b/src/spikeinterface/core/tests/test_jsonification.py
@@ -7,11 +7,7 @@ from spikeinterface.core.base import BaseExtractor
 from spikeinterface.core.core_tools import SIJsonEncoder
 from spikeinterface.core.generate import generate_recording, generate_sorting
 
-
-@pytest.fixture(scope="module")
-def numpy_generated_recording():
-    recording = generate_recording()
-    return recording
+from pathlib import Path
 
 
 @pytest.fixture(scope="module")
@@ -124,8 +120,25 @@ def test_numpy_dtype_alises_encoding():
     json.dumps(np.float32, cls=SIJsonEncoder)
 
 
-def test_recording_encoding(numpy_generated_recording):
-    recording = numpy_generated_recording
+def test_path_encoding(tmp_path):
+
+    temporary_path = tmp_path / "a_path_for_this_test"
+
+    json.dumps(temporary_path, cls=SIJsonEncoder)
+
+
+def test_path_as_annotation(tmp_path):
+    temporary_path = tmp_path / "a_path_for_this_test"
+
+    recording = generate_recording()
+    recording.annotate(path=temporary_path)
+
+    json.dumps(recording, cls=SIJsonEncoder)
+
+
+def test_recording_encoding():
+    recording = generate_recording()
+
     json.dumps(recording, cls=SIJsonEncoder)
 
 
@@ -200,4 +213,4 @@ def test_encoding_numpy_scalars_within_nested_extractors_dict(nested_extractor_d
 
 if __name__ == "__main__":
     nested_extractor = nested_extractor()
-    test_encoding_numpy_scalars_within_nested_extractors(nested_extractor_)
+    test_encoding_numpy_scalars_within_nested_extractors(nested_extractor)