internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: scalableminds/webknossos-libs

Issue:
Dataset API: Add option to rename layer
It would be helpful to be able to rename a layer of an existing dataset


Discussion:

Resolving patch:
diff --git a/wkcuber/api/layer.py b/wkcuber/api/layer.py
--- a/wkcuber/api/layer.py
+++ b/wkcuber/api/layer.py
@@ -1,5 +1,6 @@
 import logging
 import math
+import os
 from argparse import Namespace
 from shutil import rmtree
 from os.path import join
@@ -254,6 +255,31 @@ def set_bounding_box_size(self, size: Tuple[int, int, int]) -> None:
     def get_bounding_box(self) -> BoundingBox:
         return self.dataset.properties.data_layers[self.name].get_bounding_box()
 
+    def rename(self, layer_name: str) -> None:
+        """
+        Renames the layer to `layer_name`. This changes the name of the directory on disk and updates the properties.
+        """
+        assert (
+            layer_name not in self.dataset.layers.keys()
+        ), f"Failed to rename layer {self.name} to {layer_name}: The new name already exists."
+        os.rename(self.dataset.path / self.name, self.dataset.path / layer_name)
+        layer_properties = self.dataset.properties.data_layers[self.name]
+        layer_properties._name = layer_name
+        del self.dataset.properties.data_layers[self.name]
+        self.dataset.properties._data_layers[layer_name] = layer_properties
+        self.dataset.properties._export_as_json()
+        del self.dataset.layers[self.name]
+        self.dataset.layers[layer_name] = self
+        self.name = layer_name
+
+        # The MagViews need to be updated
+        for mag in self._mags.values():
+            mag.path = _find_mag_path_on_disk(self.dataset.path, self.name, mag.name)
+            if mag._is_opened:
+                # Reopen handle to dataset on disk
+                mag.close()
+                mag.open()
+
     def downsample(
         self,
         from_mag: Optional[Mag] = None,


Test patch:
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
--- a/tests/test_dataset.py
+++ b/tests/test_dataset.py
@@ -1647,3 +1647,23 @@ def test_add_copy_layer(tmp_path: Path) -> None:
 
     # Test if the changes of the properties are persisted on disk by opening it again
     assert "color" in Dataset(tmp_path / "ds").layers.keys()
+
+
+def test_rename_layer(tmp_path: Path) -> None:
+    ds = Dataset.create(tmp_path / "ds", scale=(1, 1, 1))
+    layer = ds.add_layer("color", LayerCategories.COLOR_TYPE)
+    mag = layer.add_mag(1)
+    write_data = (np.random.rand(10, 20, 30) * 255).astype(np.uint8)
+    mag.write(data=write_data)
+
+    layer.rename("color2")
+
+    assert not (tmp_path / "ds" / "color").exists()
+    assert (tmp_path / "ds" / "color2").exists()
+    assert "color2" in ds.properties.data_layers.keys()
+    assert "color2" == ds.properties.data_layers["color2"].name
+    assert "color2" in ds.layers.keys()
+    assert "color" not in ds.layers.keys()
+
+    # The "mag" object which was created before renaming the layer is still valid
+    assert np.array_equal(mag.read()[0], write_data)
Repository: scalableminds/webknossos-libs

Issue:
get_buffered_slice_writer should take Vec3IntLike for offset
currently the user has to construct Vec3Int


Discussion:

Resolving patch:
diff --git a/webknossos/webknossos/dataset/_utils/buffered_slice_writer.py b/webknossos/webknossos/dataset/_utils/buffered_slice_writer.py
--- a/webknossos/webknossos/dataset/_utils/buffered_slice_writer.py
+++ b/webknossos/webknossos/dataset/_utils/buffered_slice_writer.py
@@ -8,7 +8,7 @@
 import numpy as np
 import psutil
 
-from webknossos.geometry import Vec3Int
+from webknossos.geometry import Vec3Int, Vec3IntLike
 
 if TYPE_CHECKING:
     from webknossos.dataset import View
@@ -32,7 +32,7 @@ class BufferedSliceWriter(object):
     def __init__(
         self,
         view: "View",
-        offset: Vec3Int,
+        offset: Vec3IntLike,
         # buffer_size specifies, how many slices should be aggregated until they are flushed.
         buffer_size: int = 32,
         dimension: int = 2,  # z
@@ -48,7 +48,7 @@ def __init__(
         self.view = view
         self.buffer_size = buffer_size
         self.dtype = self.view.get_dtype()
-        self.offset = offset
+        self.offset = Vec3Int(offset)
         self.dimension = dimension
 
         assert 0 <= dimension <= 2
diff --git a/webknossos/webknossos/dataset/view.py b/webknossos/webknossos/dataset/view.py
--- a/webknossos/webknossos/dataset/view.py
+++ b/webknossos/webknossos/dataset/view.py
@@ -273,7 +273,10 @@ def get_view(
         )
 
     def get_buffered_slice_writer(
-        self, offset: Vec3Int = None, buffer_size: int = 32, dimension: int = 2  # z
+        self,
+        offset: Vec3IntLike = Vec3Int(0, 0, 0),
+        buffer_size: int = 32,
+        dimension: int = 2,  # z
     ) -> "BufferedSliceWriter":
         """
         The BufferedSliceWriter buffers multiple slices before they are written to disk.
@@ -299,15 +302,15 @@ def get_buffered_slice_writer(
 
         return BufferedSliceWriter(
             view=self,
-            offset=offset if offset is not None else Vec3Int(0, 0, 0),
+            offset=Vec3Int(offset),
             buffer_size=buffer_size,
             dimension=dimension,
         )
 
     def get_buffered_slice_reader(
         self,
-        offset: Vec3Int = None,
-        size: Vec3Int = None,
+        offset: Vec3IntLike = Vec3Int(0, 0, 0),
+        size: Optional[Vec3IntLike] = None,
         buffer_size: int = 32,
         dimension: int = 2,  # z
     ) -> "BufferedSliceReader":
@@ -333,8 +336,8 @@ def get_buffered_slice_reader(
 
         return BufferedSliceReader(
             view=self,
-            offset=offset if offset is not None else Vec3Int(0, 0, 0),
-            size=size if size is not None else self.size,
+            offset=Vec3Int(offset),
+            size=Vec3Int(size) if size is not None else self.size,
             buffer_size=buffer_size,
             dimension=dimension,
         )


Test patch:
Repository: scalableminds/webknossos-libs

Issue:
BoundingBox.contains should allow float arguments
In a recent PR `BoundingBox.contains` was changed to only accept `Vec3IntLike`. However, it makes total sense to ask whether a float position is contained in a bounding box, so the method should be reverted to accept float positions again.

For further context, see internal discussion in Slack.


Discussion:

Resolving patch:
diff --git a/webknossos/webknossos/geometry/bounding_box.py b/webknossos/webknossos/geometry/bounding_box.py
--- a/webknossos/webknossos/geometry/bounding_box.py
+++ b/webknossos/webknossos/geometry/bounding_box.py
@@ -300,9 +300,16 @@ def align_with_mag(self, mag: Mag, ceil: bool = False) -> "BoundingBox":
             bottomright = align(self.bottomright, np.floor)
         return BoundingBox(topleft, bottomright - topleft)
 
-    def contains(self, coord: Vec3IntLike) -> bool:
-
-        coord = Vec3Int(coord).to_np()
+    def contains(self, coord: Union[Vec3IntLike, np.ndarray]) -> bool:
+        """Check whether a point is inside of the bounding box.
+        Note that the point may have float coordinates in the ndarray case"""
+
+        if isinstance(coord, np.ndarray):
+            assert coord.shape == (
+                3,
+            ), f"Numpy array BoundingBox.contains must have shape (3,), got {coord.shape}."
+        else:
+            coord = Vec3Int(coord).to_np()
 
         return cast(
             bool,


Test patch:
diff --git a/webknossos/tests/test_bounding_box.py b/webknossos/tests/test_bounding_box.py
--- a/webknossos/tests/test_bounding_box.py
+++ b/webknossos/tests/test_bounding_box.py
@@ -1,3 +1,4 @@
+import numpy as np
 import pytest
 
 from webknossos.geometry import BoundingBox, Mag
@@ -59,3 +60,17 @@ def test_with_bounds() -> None:
     assert BoundingBox((1, 2, 3), (5, 5, 5)).with_bounds_z(
         new_size_z=10
     ) == BoundingBox((1, 2, 3), (5, 5, 10))
+
+
+def test_contains() -> None:
+
+    assert BoundingBox((1, 1, 1), (5, 5, 5)).contains((2, 2, 2))
+    assert BoundingBox((1, 1, 1), (5, 5, 5)).contains((1, 1, 1))
+
+    # top-left is inclusive, bottom-right is exclusive
+    assert not BoundingBox((1, 1, 1), (5, 5, 5)).contains((6, 6, 6))
+    assert not BoundingBox((1, 1, 1), (5, 5, 5)).contains((20, 20, 20))
+
+    # nd-array may contain float values
+    assert BoundingBox((1, 1, 1), (5, 5, 5)).contains(np.array([5.5, 5.5, 5.5]))
+    assert not BoundingBox((1, 1, 1), (5, 5, 5)).contains(np.array([6.0, 6.0, 6.0]))
Repository: scalableminds/webknossos-libs

Issue:
Auto-convert fails for single-page 3D tiffs
Tiff files which only have a single page (according to `Tifffile`) and still 3D cannot be converted to wkw automatically. The conversion process will properly detect that there are Z slices in z, but since there is only one page (and not Z pages) the z access is broken.

I don't know how we can use the `Tifffile` library to read the slices individually, but a simple workaround would be a preprocessing step like this:

```
from tifffile import imread, imwrite

def main():
    file_name = "problematic_file.tif"

    data = imread(file_name)
    imwrite(f"new_{file_name}", data)


if __name__ == "__main__":
    main()
```

The new file can then be converted without any problems. One could do this preprocessing step if there are more than z slices, but only one tiff page.


Discussion:

Resolving patch:
diff --git a/wkcuber/wkcuber/image_readers.py b/wkcuber/wkcuber/image_readers.py
--- a/wkcuber/wkcuber/image_readers.py
+++ b/wkcuber/wkcuber/image_readers.py
@@ -175,6 +175,29 @@ def read_dtype(self, file_name: Path) -> str:  # pylint: disable=unused-argument
 
 
 class TiffImageReader(ImageReader):
+    @staticmethod
+    def _get_page_data(tif_file: TiffFile, page_index: int) -> np.ndarray:
+        if 0 <= page_index < len(tif_file.pages):
+            return tif_file.pages[page_index].asarray()
+        elif tif_file.is_imagej:
+            # ImageJ can export 3-dimensional, single-page tiff files.
+            # In that case, we manually read from the correct filehandle
+            # by seeking to the correct position.
+            shape = tif_file.pages[0].shape
+            elements_per_page = 1
+            for i in shape:
+                elements_per_page *= i
+            series = tif_file.series[0]
+            tif_file.filehandle.seek(
+                series.offset
+                + page_index * np.dtype(series.dtype).itemsize * elements_per_page
+            )
+            data = tif_file.filehandle.read_array(series.dtype, elements_per_page)
+            data = data.reshape(shape)
+            return data
+        else:
+            raise Exception("Cannot read page data!")
+
     @staticmethod
     def _get_page_index(
         tif_file: TiffFile,
@@ -202,8 +225,8 @@ def _get_page_index(
                 page_index += s_index
         return page_index
 
+    @staticmethod
     def _find_correct_pages(
-        self,
         tif_file: TiffFile,
         channel_index: Optional[int],
         sample_index: Optional[int],
@@ -335,7 +358,7 @@ def read_array(
                 z_axis_name,
                 page_axes,
             ).items():
-                page_data = tif_file.pages[page_index].asarray()
+                page_data = self._get_page_data(tif_file, page_index)
                 # remove any axis that we do not use
                 # left over axes [(C)YX(S)] (Z axis is not on same TiffPage)
                 for axis in page_axes:


Test patch:
Repository: scalableminds/webknossos-libs

Issue:
add content to gh & pypi releases
We should add some nice content to gh & pypi releases:
1. The github tags are pushed automagically via a gh action, also updating the Changelog (which is awesome!)
2. During the release CI, we should additionally
  * add a release on github, including the aggregated changelog entries per package
  * add some content to the pypi release, maybe also updating the main page with the newest README


Discussion:

Resolving patch:
diff --git a/make_changelog.py b/_tooling/changelog_bump_version.py
similarity index 100%
rename from make_changelog.py
rename to _tooling/changelog_bump_version.py
diff --git a/check_version.py b/_tooling/check_version.py
similarity index 100%
rename from check_version.py
rename to _tooling/check_version.py


Test patch:
Repository: scalableminds/webknossos-libs

Issue:
Persist log file
When running the cuber, there usually is a lot of output (overflowing terminal buffers). It would be great if all of that output would be persisted in a log file as well to allow to easily scan and search through it, afterwards.


Discussion:
Do you have an opinion about where the log file should be placed? Something like `./logs/cuber_{timestamp}.txt` ?
Sounds good to me :)

Resolving patch:
diff --git a/webknossos/webknossos/utils.py b/webknossos/webknossos/utils.py
--- a/webknossos/webknossos/utils.py
+++ b/webknossos/webknossos/utils.py
@@ -4,6 +4,7 @@
 import json
 import logging
 import os
+import sys
 import time
 from concurrent.futures import as_completed
 from concurrent.futures._base import Future
@@ -147,10 +148,28 @@ def copy_directory_with_symlinks(
 
 
 def setup_logging(args: argparse.Namespace) -> None:
-    logging.basicConfig(
-        level=(logging.DEBUG if args.verbose else logging.INFO),
-        format="%(asctime)s %(levelname)s %(message)s",
-    )
+    log_path = Path(f"./logs/cuber_{time.strftime('%Y-%m-%d_%H%M%S')}.txt")
+
+    console_log_level = logging.DEBUG if args.verbose else logging.INFO
+    file_log_level = logging.DEBUG
+
+    logging_formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
+
+    # Always set the global log level to the more verbose of console_log_level and
+    # file_log_level to allow to log with different log levels to console and files.
+    root_logger = logging.getLogger()
+    root_logger.setLevel(min(console_log_level, file_log_level))
+
+    console = logging.StreamHandler(sys.stdout)
+    console.setLevel(console_log_level)
+    console.setFormatter(logging_formatter)
+    root_logger.addHandler(console)
+
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    file_handler = logging.FileHandler(log_path, mode="w", encoding="UTF-8")
+    file_handler.setLevel(file_log_level)
+    file_handler.setFormatter(logging_formatter)
+    root_logger.addHandler(file_handler)
 
 
 def add_verbose_flag(parser: argparse.ArgumentParser) -> None:


Test patch:
Repository: scalableminds/webknossos-libs

Issue:
Support conversion from BMP to wkw/zarr
Next to tiff and the other image formats, bmp files should also be useful and easy to support (there was at least one external user who could have used this).


Discussion:

Resolving patch:
diff --git a/wkcuber/wkcuber/image_readers.py b/wkcuber/wkcuber/image_readers.py
--- a/wkcuber/wkcuber/image_readers.py
+++ b/wkcuber/wkcuber/image_readers.py
@@ -1,14 +1,14 @@
+import logging
 from pathlib import Path
-from typing import Tuple, Dict, Union, Optional
+from typing import Dict, Optional, Tuple, Union
 
 import numpy as np
-import logging
+from czifile import CziFile
 from PIL import Image
+from tifffile import TiffFile
 
 from .vendor.dm3 import DM3
 from .vendor.dm4 import DM4File, DM4TagHeader
-from tifffile import TiffFile
-from czifile import CziFile
 
 # Disable PIL's maximum image limit.
 Image.MAX_IMAGE_PIXELS = None
@@ -779,6 +779,7 @@ def __init__(self) -> None:
             ".mj2": PillowImageReader(),
             # Other image file endings
             ".png": PillowImageReader(),
+            ".bmp": PillowImageReader(),
             ".dm3": Dm3ImageReader(),
             ".dm4": Dm4ImageReader(),
             ".czi": CziImageReader(),


Test patch:
Repository: scalableminds/webknossos-libs

Issue:
Upsampling logic for constant_z correct?
I would like to upsample a segmentation from 4-4-1 to 2-2-1. A minimal example would look similar to:
```python
import tempfile
import numpy as np
from webknossos import Mag, Dataset

def main():
    data = np.ones((3,4,5), dtype=np.uint8)
    scale = (128, 128, 35)
    from_mag = Mag('4-4-1')

    with tempfile.TemporaryDirectory() as path:
        layer = Dataset(path, scale=scale)\
            .get_or_add_layer('test_seg', 'segmentation', largest_segment_id=1)

        layer.get_or_add_mag(from_mag).write(data)

        layer.upsample(
            from_mag = from_mag,
            min_mag = Mag('2-2-1'),
            compress=False,
            sampling_mode = 'constant_z',
        )

if __name__ == '__main__':
    main()
```
I get the error message:
```
RuntimeError: The maximum mag 4-4-1 can
not be reached from 2-2-1 with scale (128.0, 128.0, 35.0)!
```
I do not see how 4-4-1 shouldn't be reachable from 2-2-1. (Note the switched target <-> source mag order due to the reuse of the downsampling code).  Together with the comment:
https://github.com/scalableminds/webknossos-libs/blob/7777ca18f8bab66a1f34631ba75fc6e2947d4566/webknossos/webknossos/dataset/downsampling_utils.py#L49-L51
I concluded that either the comment is wrong or there is a hidden logic error because:

https://github.com/scalableminds/webknossos-libs/blob/0713b80d3dbb9d0ce9575fbaf2ff47644eff9d64/webknossos/webknossos/dataset/layer.py#L919-L927
Since `voxel_size` equals `scale` in the presented case. 

Tested with webknossos `0.9.21`.


Discussion:
Hi and thanks for the detailed report! I agree that this should work as you describe. We will investigate what is going on here.

Resolving patch:
diff --git a/webknossos/webknossos/dataset/layer.py b/webknossos/webknossos/dataset/layer.py
--- a/webknossos/webknossos/dataset/layer.py
+++ b/webknossos/webknossos/dataset/layer.py
@@ -924,7 +924,7 @@ def upsample(
             finest_mag_with_fixed_z = finest_mag.to_list()
             finest_mag_with_fixed_z[2] = from_mag.to_list()[2]
             finest_mag = Mag(finest_mag_with_fixed_z)
-            voxel_size = self.dataset.voxel_size
+            voxel_size = None
         else:
             raise AttributeError(
                 f"Upsampling failed: {sampling_mode} is not a valid UpsamplingMode ({SamplingModes.ANISOTROPIC}, {SamplingModes.ISOTROPIC}, {SamplingModes.CONSTANT_Z})"


Test patch:
Repository: scalableminds/webknossos-libs

Issue:
auto-detect conversion should allow --pad
the --pad option is available for convert_image_stack (I think?) but not for the auto-detect converter. Thus, it is not available for the wk-worker


Discussion:
Instead of adding `--pad`, we might just use the new `add_layer_from_images` (PR #741) and use it instead of the custom logic.

Resolving patch:
diff --git a/wkcuber/wkcuber/converter.py b/wkcuber/wkcuber/converter.py
--- a/wkcuber/wkcuber/converter.py
+++ b/wkcuber/wkcuber/converter.py
@@ -48,6 +48,15 @@ def create_parser() -> ArgumentParser:
         type=parse_path,
     )
 
+    parser.add_argument(
+        "--pad",
+        help="Automatically pad image files at the bottom and right borders. "
+        "Use this, when the input images don't have a common size, but have "
+        "their origin at (0, 0).",
+        default=False,
+        action="store_true",
+    )
+
     add_voxel_size_flag(parser)
     add_verbose_flag(parser)
     add_data_format_flags(parser)


Test patch:
Repository: scalableminds/webknossos-libs

Issue:
Add axis_order for zarr datasets
## Detailed Description
- Affected library: webknossos

## Use Cases & Context
Webknossos requires `axis_order` fields for each mag in the datasource-properties.json for Zarr datasets. The Python library should write these out:
```json
"axisOrder": {
  "x": 1,
  "y": 2,
  "z": 3,
  "c": 0
}
```



Discussion:

Resolving patch:
diff --git a/webknossos/webknossos/dataset/layer.py b/webknossos/webknossos/dataset/layer.py
--- a/webknossos/webknossos/dataset/layer.py
+++ b/webknossos/webknossos/dataset/layer.py
@@ -415,6 +415,11 @@ def add_mag(
                     if mag_array_info.data_format == DataFormat.WKW
                     else None
                 ),
+                axis_order=(
+                    {"x": 1, "y": 2, "z": 3, "c": 0}
+                    if mag_array_info.data_format == DataFormat.Zarr
+                    else None
+                ),
             )
         ]
 
@@ -447,6 +452,11 @@ def add_mag_for_existing_files(
                     if mag_array_info.data_format == DataFormat.WKW
                     else None
                 ),
+                axis_order=(
+                    {"x": 1, "y": 2, "z": 3, "c": 0}
+                    if mag_array_info.data_format == DataFormat.Zarr
+                    else None
+                ),
             )
         )
         self.dataset._export_as_json()
diff --git a/webknossos/webknossos/dataset/properties.py b/webknossos/webknossos/dataset/properties.py
--- a/webknossos/webknossos/dataset/properties.py
+++ b/webknossos/webknossos/dataset/properties.py
@@ -121,6 +121,7 @@ class LayerViewConfiguration:
 class MagViewProperties:
     mag: Mag
     cube_length: Optional[int] = None
+    axis_order: Optional[Dict[str, int]] = None
 
     @property
     def resolution(self) -> Mag:


Test patch:
Repository: scalableminds/webknossos-libs

Issue:
Dataset.from_images should support NIFTI, recognize .nii
`.nii` is an alternative file ending for nifti. The wkcuber CLI currrently prints `Converting Nifti dataset`, but then `Could not find any supported image data. The following suffixes are supported …`

I think nifti used to be supported by the wkcuber


Discussion:

Resolving patch:
diff --git a/webknossos/webknossos/dataset/_utils/pims_images.py b/webknossos/webknossos/dataset/_utils/pims_images.py
--- a/webknossos/webknossos/dataset/_utils/pims_images.py
+++ b/webknossos/webknossos/dataset/_utils/pims_images.py
@@ -725,6 +725,7 @@ def get_valid_bioformats_suffixes() -> Set[str]:
         "raw",
         "xml",
         "gif",
+        "nii",
     }
 
 


Test patch:
Repository: scalableminds/webknossos-libs

Issue:
upload command on windows: backslashes on server, invalid dataset
A user created a valid dataset on a windows machine with the `webknossos convert` command, then called `webknossos upload` with a valid token. The upload went through, but the directory structure got lost: the files on the server had backslashes in the paths, like `'color\2-2-1\z0\y7\x1.wkw'`. Instead, when sending files to upload, the client should always replace the client’s path separator by `/`.


Discussion:

Resolving patch:
diff --git a/webknossos/webknossos/client/_upload_dataset.py b/webknossos/webknossos/client/_upload_dataset.py
--- a/webknossos/webknossos/client/_upload_dataset.py
+++ b/webknossos/webknossos/client/_upload_dataset.py
@@ -152,7 +152,7 @@ def upload_dataset(
             },
             chunk_size=100 * 1024 * 1024,  # 100 MiB
             generate_unique_identifier=lambda _,
-            relative_path: f"{upload_id}/{relative_path}",
+            relative_path: f"{upload_id}/{relative_path.as_posix()}",
             test_chunks=False,
             permanent_errors=[400, 403, 404, 409, 415, 500, 501],
             client=httpx.Client(timeout=None),


Test patch: