Repository: scalableminds/webknossos-libs

Issue:
executor arg should be optional
I was just using the `MagView.for_each_chunk` function, which  is a great utility. However, I would like that the `executor` arg is optional. If it is `None`, a simple for loop should be used.

Also, a few comments about the code  sample in the docs:

    The view is chunked into multiple sub-views of size `chunk_size`.
    Then, `work_on_chunk` is performed on each sub-view.
    Besides the view, the counter 'i' is passed to the 'work_on_chunk',
    which can be used for logging. Additional parameter for 'work_on_chunk' can be specified.
    The computation of each chunk has to be independent of each other.
    Therefore, the work can be parallelized with `executor`.

    If the `View` is of type `MagView`, only the bounding box from the properties is chunked.

    Example:
    ```python
    from wkcuber.utils import get_executor_for_args, named_partial

    def some_work(args: Tuple[View, int], some_parameter: int) -> None:
        view_of_single_chunk, i = args
        # perform operations on the view
        ...

    # ...
    # let 'mag1' be a `MagView`
    view = mag1.get_view()
    with get_executor_for_args(None) as executor:
        func = named_partial(advanced_chunk_job, some_parameter=42)
        view.for_each_chunk(
            func,
            chunk_size=(100, 100, 100),  # Use mag1._get_file_dimensions() if the size of the chunks should match the size of the files on disk
            executor=executor,
        )
    ```

* `get_executor_for_args`  shouldn't be used in a sample, because it is hard to construct for end-users in a script
* the argument is called `work_on_chunk`. the sample function is  called `some_work`.  it is invoked as `advanced_chunk_job` (bug). maybe this can be unified


Discussion:

Resolving patch:
diff --git a/webknossos/webknossos/dataset/view.py b/webknossos/webknossos/dataset/view.py
--- a/webknossos/webknossos/dataset/view.py
+++ b/webknossos/webknossos/dataset/view.py
@@ -296,7 +296,9 @@ def for_each_chunk(
         self,
         work_on_chunk: Callable[[Tuple["View", int]], None],
         chunk_size: Tuple[int, int, int],
-        executor: Union[ClusterExecutor, cluster_tools.WrappedProcessPoolExecutor],
+        executor: Optional[
+            Union[ClusterExecutor, cluster_tools.WrappedProcessPoolExecutor]
+        ] = None,
     ) -> None:
         """
         The view is chunked into multiple sub-views of size `chunk_size`.
@@ -320,13 +322,11 @@ def some_work(args: Tuple[View, int], some_parameter: int) -> None:
         # ...
         # let 'mag1' be a `MagView`
         view = mag1.get_view()
-        with get_executor_for_args(None) as executor:
-            func = named_partial(advanced_chunk_job, some_parameter=42)
-            view.for_each_chunk(
-                func,
-                chunk_size=(100, 100, 100),  # Use mag1._get_file_dimensions() if the size of the chunks should match the size of the files on disk
-                executor=executor,
-            )
+        func = named_partial(some_work, some_parameter=42)
+        view.for_each_chunk(
+            func,
+            chunk_size=(100, 100, 100),  # Use mag1._get_file_dimensions() if the size of the chunks should match the size of the files on disk
+        )
         ```
         """
 
@@ -354,7 +354,11 @@ def some_work(args: Tuple[View, int], some_parameter: int) -> None:
             job_args.append((chunk_view, i))
 
         # execute the work for each chunk
-        wait_and_ensure_success(executor.map_to_futures(work_on_chunk, job_args))
+        if executor is None:
+            for args in job_args:
+                work_on_chunk(args)
+        else:
+            wait_and_ensure_success(executor.map_to_futures(work_on_chunk, job_args))
 
     def for_zipped_chunks(
         self,
@@ -362,7 +366,9 @@ def for_zipped_chunks(
         target_view: "View",
         source_chunk_size: Vec3,
         target_chunk_size: Vec3,
-        executor: Union[ClusterExecutor, cluster_tools.WrappedProcessPoolExecutor],
+        executor: Optional[
+            Union[ClusterExecutor, cluster_tools.WrappedProcessPoolExecutor]
+        ] = None,
     ) -> None:
         """
         This method is similar to 'for_each_chunk' in the sense, that it delegates work to smaller chunks.
@@ -439,7 +445,11 @@ def for_zipped_chunks(
             job_args.append((source_chunk_view, target_chunk_view, i))
 
         # execute the work for each pair of chunks
-        wait_and_ensure_success(executor.map_to_futures(work_on_chunk, job_args))
+        if executor is None:
+            for args in job_args:
+                work_on_chunk(args)
+        else:
+            wait_and_ensure_success(executor.map_to_futures(work_on_chunk, job_args))
 
     def _is_compressed(self) -> bool:
         return (


Test patch:
diff --git a/webknossos/tests/test_dataset.py b/webknossos/tests/test_dataset.py
--- a/webknossos/tests/test_dataset.py
+++ b/webknossos/tests/test_dataset.py
@@ -620,13 +620,23 @@ def test_chunking_wk(tmp_path: Path) -> None:
     original_data = (np.random.rand(50, 100, 150) * 205).astype(np.uint8)
     mag.write(offset=(70, 80, 90), data=original_data)
 
+    # Test with executor
     with get_executor_for_args(None) as executor:
         mag.for_each_chunk(
             chunk_job,
             chunk_size=(64, 64, 64),
             executor=executor,
         )
+    assert np.array_equal(original_data + 50, mag.get_view().read()[0])
+
+    # Reset the data
+    mag.write(offset=(70, 80, 90), data=original_data)
 
+    # Test without executor
+    mag.for_each_chunk(
+        chunk_job,
+        chunk_size=(64, 64, 64),
+    )
     assert np.array_equal(original_data + 50, mag.get_view().read()[0])
 
     assure_exported_properties(ds)