Repository: scalableminds/webknossos-libs

Issue:
webknossos new CLI JSON error with multiprocessing futures
## Context
- webknossos CLI version 0.13.0
- cubing tiff series of 5714 images, images require padding, total size 121G
- the same series cubes without error in an older wkcuber version (0.9.21)

## Expected Behavior
- cubing should proceed and create mag1 wkw dataset without errors

## Current Behavior

I ran from command line using this command:
```
webknossos convert --jobs 72 --voxel-size 256,256,35 --compress --name Retina_macaque_MEA-align2-regions /gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Retina_macaque_MEA/meta/webknossos/Retina_macaque_MEA-align2-regions-tiffs /gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Retina_macaque_MEA/meta/webknossos/Retina_macaque_MEA-align2-regions
```

It results in this error and terminates before any cubing occurs:
```
ERROR:root:A future crashed with an exception: Expecting value: line 1 column 1 (char 0). Future: <Future at 0x2b6ccbd4a730 state=finished raised JSONDecodeError>
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new/lib/python3.9/site-packages/webknossos/cli/co │
│ nvert.py:95 in main                                                                              │
│                                                                                                  │
│    92 │   )                                                                                      │
│    93 │                                                                                          │
│    94 │   with get_executor_for_args(args=executor_args) as executor:                            │
│ ❱  95 │   │   dataset = Dataset.from_images(                                                     │
│    96 │   │   │   source,                                                                        │
│    97 │   │   │   target,                                                                        │
│    98 │   │   │   voxel_size,                                                                    │
│                                                                                                  │
│ ╭─────────────────────────────────────────── locals ───────────────────────────────────────────╮ │
│ │              compress = True                                                                 │ │
│ │           data_format = <DataFormat.WKW: 'wkw'>                                              │ │
│ │ distribution_strategy = <DistributionStrategy.MULTIPROCESSING: 'multiprocessing'>            │ │
│ │              executor = <cluster_tools.executors.multiprocessing.MultiprocessingExecutor     │ │
│ │                         object at 0x2b6cbb79c4c0>                                            │ │
│ │         executor_args = Namespace(jobs=72,                                                   │ │
│ │                         distribution_strategy=<DistributionStrategy.MULTIPROCESSING:         │ │
│ │                         'multiprocessing'>, job_resources=None)                              │ │
│ │         job_resources = None                                                                 │ │
│ │                  jobs = 72                                                                   │ │
│ │                  name = 'Retina_macaque_MEA-align2-regions'                                  │ │
│ │                source = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Retina_ma… │ │
│ │                target = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Retina_ma… │ │
│ │            voxel_size = VoxelSize(x=256.0, y=256.0, z=35.0)                                  │ │
│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new/lib/python3.9/site-packages/webknossos/datase │
│ t/dataset.py:640 in from_images                                                                  │
│                                                                                                  │
│    637 │   │   │   │   )                                                                         │
│    638 │   │   │   else:                                                                         │
│    639 │   │   │   │   category = layer_category                                                 │
│ ❱  640 │   │   │   ds.add_layer_from_images(                                                     │
│    641 │   │   │   │   filepaths[0] if len(filepaths) == 1 else filepaths,                       │
│    642 │   │   │   │   layer_name,                                                               │
│    643 │   │   │   │   category=category,                                                        │
│                                                                                                  │
│ ╭─────────────────────────────────────────── locals ───────────────────────────────────────────╮ │
│ │                 batch_size = None                                                            │ │
│ │                   category = 'color'                                                         │ │
│ │                chunk_shape = None                                                            │ │
│ │           chunks_per_shard = None                                                            │ │
│ │                        cls = <class 'webknossos.dataset.dataset.Dataset'>                    │ │
│ │                   compress = False                                                           │ │
│ │                data_format = <DataFormat.WKW: 'wkw'>                                         │ │
│ │                         ds = Dataset(PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/brigg… │ │
│ │                   executor = <cluster_tools.executors.multiprocessing.MultiprocessingExecut… │ │
│ │                              object at 0x2b6cbb79c4c0>                                       │ │
│ │                  filepaths = [                                                               │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   ... +5704                                                   │ │
│ │                              ]                                                               │ │
│ │        filepaths_per_layer = {                                                               │ │
│ │                              │   'Retina_macaque_MEA-align2-regions-tiffs': [                │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │   ... +5704                                               │ │
│ │                              │   ]                                                           │ │
│ │                              }                                                               │ │
│ │                     flip_x = False                                                           │ │
│ │                     flip_y = False                                                           │ │
│ │                     flip_z = False                                                           │ │
│ │    get_valid_pims_suffixes = <function get_valid_pims_suffixes at 0x2b6ccbbbd3a0>            │ │
│ │                 input_file = PosixPath('wafer03_manifest01329_1330_S1330R1330_stitched.tiff… │ │
│ │                input_files = [                                                               │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer03_manifest00684_0685_S685R685_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer02_manifest00371_0372_S372R372_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer01_manifest01034_1035_S1035R1035_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer01_manifest01487_1488_S1488R1488_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer03_manifest00411_0412_S412R412_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer01_manifest01511_1512_S1512R1512_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer02_manifest01139_1140_S1140R1140_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer01_manifest00815_0816_S816R816_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer03_manifest01307_1308_S1308R1308_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer03_manifest00894_0895_S895R895_stitched.tiff'), │ │
│ │                              │   ... +5704                                                   │ │
│ │                              ]                                                               │ │
│ │                 input_path = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                input_upath = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │             layer_category = None                                                            │ │
│ │                 layer_name = 'Retina_macaque_MEA-align2-regions-tiffs'                       │ │
│ │ map_filepath_to_layer_name = <function                                                       │ │
│ │                              Dataset.ConversionLayerMapping._to_callable.<locals>.<lambda>   │ │
│ │                              at 0x2b6ccbbbd670>                                              │ │
│ │                 max_layers = 20                                                              │ │
│ │                       name = 'Retina_macaque_MEA-align2-regions'                             │ │
│ │                output_path = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                    swap_xy = False                                                           │ │
│ │             use_bioformats = None                                                            │ │
│ │             valid_suffixes = {                                                               │ │
│ │                              │   'gdal',                                                     │ │
│ │                              │   'jpeg',                                                     │ │
│ │                              │   'gif',                                                      │ │
│ │                              │   'ppmraw',                                                   │ │
│ │                              │   'pgm',                                                      │ │
│ │                              │   'ics',                                                      │ │
│ │                              │   'ipl',                                                      │ │
│ │                              │   'stk',                                                      │ │
│ │                              │   'mp4',                                                      │ │
│ │                              │   'avi',                                                      │ │
│ │                              │   ... +59                                                     │ │
│ │                              }                                                               │ │
│ │                 voxel_size = VoxelSize(x=256.0, y=256.0, z=35.0)                             │ │
│ │          z_slices_sort_key = functools.partial(<function natsort_key at 0x2b6e9d47cd30>,     │ │
│ │                              key=None, string_func=<function                                 │ │
│ │                              parse_string_factory.<locals>.func at 0x2b6f26e68310>,          │ │
│ │                              bytes_func=<function parse_bytes_factory.<locals>.<lambda> at   │ │
│ │                              0x2b6f26e683a0>, num_func=<function                             │ │
│ │                              parse_number_factory.<locals>.func at 0x2b6f26e68430>)          │ │
│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new/lib/python3.9/site-packages/webknossos/datase │
│ t/dataset.py:1234 in add_layer_from_images                                                       │
│                                                                                                  │
│   1231 │   │   │   │   │   module="webknossos",                                                  │
│   1232 │   │   │   │   )                                                                         │
│   1233 │   │   │   │   with get_executor_for_args(None, executor) as executor:                   │
│ ❱ 1234 │   │   │   │   │   shapes_and_max_ids = wait_and_ensure_success(                         │
│   1235 │   │   │   │   │   │   executor.map_to_futures(func_per_chunk, args),                    │
│   1236 │   │   │   │   │   │   progress_desc="Creating layer from images",                       │
│   1237 │   │   │   │   │   )                                                                     │
│                                                                                                  │
│ ╭─────────────────────────────────────────── locals ───────────────────────────────────────────╮ │
│ │                                      _ = 0                                                   │ │
│ │                       add_layer_kwargs = {}                                                  │ │
│ │                  allow_multiple_layers = True                                                │ │
│ │                                   args = [                                                   │ │
│ │                                          │   (0, 32),                                        │ │
│ │                                          │   (32, 64),                                       │ │
│ │                                          │   (64, 96),                                       │ │
│ │                                          │   (96, 128),                                      │ │
│ │                                          │   (128, 160),                                     │ │
│ │                                          │   (160, 192),                                     │ │
│ │                                          │   (192, 224),                                     │ │
│ │                                          │   (224, 256),                                     │ │
│ │                                          │   (256, 288),                                     │ │
│ │                                          │   (288, 320),                                     │ │
│ │                                          │   ... +169                                        │ │
│ │                                          ]                                                   │ │
│ │                             batch_size = 32                                                  │ │
│ │                               category = 'color'                                             │ │
│ │                                channel = None                                                │ │
│ │                            chunk_shape = None                                                │ │
│ │                             chunk_size = None                                                │ │
│ │                       chunks_per_shard = None                                                │ │
│ │                               compress = False                                               │ │
│ │                          current_dtype = dtype('uint8')                                      │ │
│ │                            czi_channel = None                                                │ │
│ │                            data_format = <DataFormat.WKW: 'wkw'>                             │ │
│ │                            dimwise_max = <function dimwise_max at 0x2b6cbb7b83a0>            │ │
│ │                                  dtype = None                                                │ │
│ │                               executor = <cluster_tools.executors.multiprocessing.Multiproc… │ │
│ │                                          object at 0x2b6cbb79c4c0>                           │ │
│ │                            first_layer = None                                                │ │
│ │                                 flip_x = False                                               │ │
│ │                                 flip_y = False                                               │ │
│ │                                 flip_z = False                                               │ │
│ │                         func_per_chunk = functools.partial(<bound method                     │ │
│ │                                          PimsImages.copy_to_view of                          │ │
│ │                                          <webknossos.dataset._utils.pims_images.PimsImages   │ │
│ │                                          object at 0x2b6cab443e50>>,                         │ │
│ │                                          mag_view=MagView(name='1',                          │ │
│ │                                          bounding_box=BoundingBox(topleft=(0, 0, 0),         │ │
│ │                                          size=(4956, 5459, 5714))), is_segmentation=False,   │ │
│ │                                          dtype=dtype('uint8'))                               │ │
│ │                                 images = [                                                   │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │   ... +5704                                       │ │
│ │                                          ]                                                   │ │
│ │                                  layer = Layer('Retina_macaque_MEA-align2-regions-tiffs',    │ │
│ │                                          dtype_per_channel=uint8, num_channels=1)            │ │
│ │                             layer_name = 'Retina_macaque_MEA-align2-regions-tiffs'           │ │
│ │                      layer_name_suffix = ''                                                  │ │
│ │                                    mag = Mag(_mag=Vec3Int(1,1,1))                            │ │
│ │                               mag_view = MagView(name='1',                                   │ │
│ │                                          bounding_box=BoundingBox(topleft=(0, 0, 0),         │ │
│ │                                          size=(4956, 5459, 5714)))                           │ │
│ │                             max_layers = 20                                                  │ │
│ │                            pims_images = <webknossos.dataset._utils.pims_images.PimsImages   │ │
│ │                                          object at 0x2b6cab443e50>                           │ │
│ │                       pims_open_kwargs = {}                                                  │ │
│ │                             PimsImages = <class                                              │ │
│ │                                          'webknossos.dataset._utils.pims_images.PimsImages'> │ │
│ │                        possible_layers = None                                                │ │
│ │                                   self = Dataset(PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-pro… │ │
│ │ suffix_with_pims_open_kwargs_per_layer = {'': {}}                                            │ │
│ │                                swap_xy = False                                               │ │
│ │                              timepoint = None                                                │ │
│ │                                topleft = Vec3Int(0,0,0)                                      │ │
│ │                   truncate_rgba_to_rgb = True                                                │ │
│ │                         use_bioformats = None                                                │ │
│ │                                  z_end = 5714                                                │ │
│ │                                z_start = 5696                                                │ │
│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new/lib/python3.9/site-packages/webknossos/utils. │
│ py:137 in wait_and_ensure_success                                                                │
│                                                                                                  │
│   134 │   │   with get_rich_progress() as progress:                                              │
│   135 │   │   │   task = progress.add_task(progress_desc, total=len(futures))                    │
│   136 │   │   │   for fut in as_completed(futures):                                              │
│ ❱ 137 │   │   │   │   results.append(fut.result())                                               │
│   138 │   │   │   │   progress.update(task, advance=1)                                           │
│   139 │   return results                                                                         │
│   140                                                                                            │
│                                                                                                  │
│ ╭───────────────────────────────────── locals ─────────────────────────────────────╮             │
│ │           fut = <Future at 0x2b6ccbd01730 state=finished raised JSONDecodeError> │             │
│ │       futures = [                                                                │             │
│ │                 │   <Future at 0x2b6cd3e123d0 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b6ccbbda1f0 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b6cd3e12460 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b6cd3e12910 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b6cd3e12be0 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b6cd3e129a0 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b6cd3eb8e50 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b6cd3eb8370 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b6cd3eb8f70 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b6cd3eb8c40 state=finished returned tuple>,    │             │
│ │                 │   ... +169                                                     │             │
│ │                 ]                                                                │             │
│ │      progress = <rich.progress.Progress object at 0x2b6cd3d5d850>                │             │
│ │ progress_desc = 'Creating layer from images'                                     │             │
│ │       results = []                                                               │             │
│ │          task = 0                                                                │             │
│ ╰──────────────────────────────────────────────────────────────────────────────────╯             │
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new/lib/python3.9/concurrent/futures/_base.py:439 │
│ in result                                                                                        │
│                                                                                                  │
│   436 │   │   │   │   if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:                     │
│   437 │   │   │   │   │   raise CancelledError()                                                 │
│   438 │   │   │   │   elif self._state == FINISHED:                                              │
│ ❱ 439 │   │   │   │   │   return self.__get_result()                                             │
│   440 │   │   │   │                                                                              │
│   441 │   │   │   │   self._condition.wait(timeout)                                              │
│   442                                                                                            │
│                                                                                                  │
│ ╭──── locals ────╮                                                                               │
│ │    self = None │                                                                               │
│ │ timeout = None │                                                                               │
│ ╰────────────────╯                                                                               │
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new/lib/python3.9/concurrent/futures/_base.py:391 │
│ in __get_result                                                                                  │
│                                                                                                  │
│   388 │   def __get_result(self):                                                                │
│   389 │   │   if self._exception:                                                                │
│   390 │   │   │   try:                                                                           │
│ ❱ 391 │   │   │   │   raise self._exception                                                      │
│   392 │   │   │   finally:                                                                       │
│   393 │   │   │   │   # Break a reference cycle with the exception in self._exception            │
│   394 │   │   │   │   self = None                                                                │
│                                                                                                  │
│ ╭── locals ───╮                                                                                  │
│ │ self = None │                                                                                  │
│ ╰─────────────╯                                                                                  │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
JSONDecodeError: Expecting value: line 1 column 1 (char 0)
```

I ran the cubing to completion using an older version of the wkcuber CLI (0.9.21), which I modified so that it supports the  `--pad` argument with this command line:
```
python -m wkcuber --jobs 72 --scale 256,256,35 --max_mag 32 --sampling_mode constant_z --pad --name Retina_macaque_MEA-align2-regions /gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Retina_macaque_MEA/meta/webknossos/Retina_macaque_MEA-align2-regions-tiffs /gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Retina_macaque_MEA/meta/webknossos/Retina_macaque_MEA-align2-regions
```

## Steps to Reproduce the bug

It's not clear to me the source of the issue, but unless it's something specific about my environment or some specific combination of options, it seems to me that just running `webknossos convert` using multiprocessing on a tiff series should reproduce the error.

## Your Environment for bug

```
(wkw-new) [watkins@somalogin01 cubing]$ python --version
Python 3.9.16
(wkw-new) [watkins@somalogin01 cubing]$ pip list installed | grep webknossos
webknossos                0.13.0
(wkw-new) [watkins@somalogin01 cubing]$ uname -a
Linux somalogin01 3.10.0-1160.el7.x86_64 #1 SMP Mon Oct 19 16:18:59 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux
```

webknossos CLI was installed with:
```
pip install "webknossos[all]"
```



Discussion:
Thanks for reporting this. We will investigate the issue and report back.
Hi @elhuhdron, could you try running the command with the version in this branch [`Fix_parallel_BufferedSliceWriter_bug`](https://github.com/scalableminds/webknossos-libs/tree/Fix_parallel_BufferedSliceWriter_bug)? We added some more exception logging to figure out where the JSON error is coming from. Thanks!
Hi @normanrz,
Sure, no problem. As an aside, I tried a few things before figuring out how to install from a local clone, and settled on this:

```
pip install --prefer-binary /gpfs/soma_fs/home/watkins/gits/webknossos-libs/webknossos[all]
```

It worked in the sense of installing and also I can see the latest change to `utils.py` from the top commit for this branch. Also it seems to uninstall and reinstall `cluster-tools` which I assume then comes from `PyPI`, meaning that any local changes to `cluster-tools` would not be installed. It seems to me that this does not matter in this case, but would you suggest a better / different method? I did also try `poetry install` in each of the subdirectories, but this resulted in missing dependencies when I ran the cuber.

After the successful install, I ran the same command again in the new env and it did create a different error message, but maybe this is not what was intended?
```
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new/lib/python3.9/site-packages/webknossos/cli/co │
│ nvert.py:95 in main                                                                              │
│                                                                                                  │
│    92 │   )                                                                                      │
│    93 │                                                                                          │
│    94 │   with get_executor_for_args(args=executor_args) as executor:                            │
│ ❱  95 │   │   dataset = Dataset.from_images(                                                     │
│    96 │   │   │   source,                                                                        │
│    97 │   │   │   target,                                                                        │
│    98 │   │   │   voxel_size,                                                                    │
│                                                                                                  │
│ ╭─────────────────────────────────────────── locals ───────────────────────────────────────────╮ │
│ │              compress = True                                                                 │ │
│ │           data_format = <DataFormat.WKW: 'wkw'>                                              │ │
│ │ distribution_strategy = <DistributionStrategy.MULTIPROCESSING: 'multiprocessing'>            │ │
│ │              executor = <cluster_tools.executors.multiprocessing.MultiprocessingExecutor     │ │
│ │                         object at 0x2b9f25d894c0>                                            │ │
│ │         executor_args = Namespace(jobs=48,                                                   │ │
│ │                         distribution_strategy=<DistributionStrategy.MULTIPROCESSING:         │ │
│ │                         'multiprocessing'>, job_resources=None)                              │ │
│ │         job_resources = None                                                                 │ │
│ │                  jobs = 48                                                                   │ │
│ │                  name = 'Retina_macaque_MEA-align2-regions'                                  │ │
│ │                source = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Retina_ma… │ │
│ │                target = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Retina_ma… │ │
│ │            voxel_size = VoxelSize(x=256.0, y=256.0, z=35.0)                                  │ │
│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new/lib/python3.9/site-packages/webknossos/datase │
│ t/dataset.py:640 in from_images                                                                  │
│                                                                                                  │
│    637 │   │   │   │   )                                                                         │
│    638 │   │   │   else:                                                                         │
│    639 │   │   │   │   category = layer_category                                                 │
│ ❱  640 │   │   │   ds.add_layer_from_images(                                                     │
│    641 │   │   │   │   filepaths[0] if len(filepaths) == 1 else filepaths,                       │
│    642 │   │   │   │   layer_name,                                                               │
│    643 │   │   │   │   category=category,                                                        │
│                                                                                                  │
│ ╭─────────────────────────────────────────── locals ───────────────────────────────────────────╮ │
│ │                 batch_size = None                                                            │ │
│ │                   category = 'color'                                                         │ │
│ │                chunk_shape = None                                                            │ │
│ │           chunks_per_shard = None                                                            │ │
│ │                        cls = <class 'webknossos.dataset.dataset.Dataset'>                    │ │
│ │                   compress = False                                                           │ │
│ │                data_format = <DataFormat.WKW: 'wkw'>                                         │ │
│ │                         ds = Dataset(PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/brigg… │ │
│ │                   executor = <cluster_tools.executors.multiprocessing.MultiprocessingExecut… │ │
│ │                              object at 0x2b9f25d894c0>                                       │ │
│ │                  filepaths = [                                                               │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   ... +5704                                                   │ │
│ │                              ]                                                               │ │
│ │        filepaths_per_layer = {                                                               │ │
│ │                              │   'Retina_macaque_MEA-align2-regions-tiffs': [                │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                              │   │   ... +5704                                               │ │
│ │                              │   ]                                                           │ │
│ │                              }                                                               │ │
│ │                     flip_x = False                                                           │ │
│ │                     flip_y = False                                                           │ │
│ │                     flip_z = False                                                           │ │
│ │    get_valid_pims_suffixes = <function get_valid_pims_suffixes at 0x2b9f361a75e0>            │ │
│ │                 input_file = PosixPath('wafer03_manifest01329_1330_S1330R1330_stitched.tiff… │ │
│ │                input_files = [                                                               │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer03_manifest00684_0685_S685R685_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer02_manifest00371_0372_S372R372_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer01_manifest01034_1035_S1035R1035_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer01_manifest01487_1488_S1488R1488_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer03_manifest00411_0412_S412R412_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer01_manifest01511_1512_S1512R1512_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer02_manifest01139_1140_S1140R1140_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer01_manifest00815_0816_S816R816_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer03_manifest01307_1308_S1308R1308_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer03_manifest00894_0895_S895R895_stitched.tiff'), │ │
│ │                              │   ... +5704                                                   │ │
│ │                              ]                                                               │ │
│ │                 input_path = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                input_upath = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │             layer_category = None                                                            │ │
│ │                 layer_name = 'Retina_macaque_MEA-align2-regions-tiffs'                       │ │
│ │ map_filepath_to_layer_name = <function                                                       │ │
│ │                              Dataset.ConversionLayerMapping._to_callable.<locals>.<lambda>   │ │
│ │                              at 0x2b9f361a78b0>                                              │ │
│ │                 max_layers = 20                                                              │ │
│ │                       name = 'Retina_macaque_MEA-align2-regions'                             │ │
│ │                output_path = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/briggman/Reti… │ │
│ │                    swap_xy = False                                                           │ │
│ │             use_bioformats = None                                                            │ │
│ │             valid_suffixes = {                                                               │ │
│ │                              │   'tif',                                                      │ │
│ │                              │   'lif',                                                      │ │
│ │                              │   'ppmraw',                                                   │ │
│ │                              │   'pcx',                                                      │ │
│ │                              │   'png',                                                      │ │
│ │                              │   'dm4',                                                      │ │
│ │                              │   'mkv',                                                      │ │
│ │                              │   'avbin',                                                    │ │
│ │                              │   'webp',                                                     │ │
│ │                              │   'wbmp',                                                     │ │
│ │                              │   ... +59                                                     │ │
│ │                              }                                                               │ │
│ │                 voxel_size = VoxelSize(x=256.0, y=256.0, z=35.0)                             │ │
│ │          z_slices_sort_key = functools.partial(<function natsort_key at 0x2ba02c42be50>,     │ │
│ │                              key=None, string_func=<function                                 │ │
│ │                              parse_string_factory.<locals>.func at 0x2ba093da6940>,          │ │
│ │                              bytes_func=<function parse_bytes_factory.<locals>.<lambda> at   │ │
│ │                              0x2ba093da69d0>, num_func=<function                             │ │
│ │                              parse_number_factory.<locals>.func at 0x2ba093da6a60>)          │ │
│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new/lib/python3.9/site-packages/webknossos/datase │
│ t/dataset.py:1234 in add_layer_from_images                                                       │
│                                                                                                  │
│   1231 │   │   │   │   │   module="webknossos",                                                  │
│   1232 │   │   │   │   )                                                                         │
│   1233 │   │   │   │   with get_executor_for_args(None, executor) as executor:                   │
│ ❱ 1234 │   │   │   │   │   shapes_and_max_ids = wait_and_ensure_success(                         │
│   1235 │   │   │   │   │   │   executor.map_to_futures(func_per_chunk, args),                    │
│   1236 │   │   │   │   │   │   progress_desc="Creating layer from images",                       │
│   1237 │   │   │   │   │   )                                                                     │
│                                                                                                  │
│ ╭─────────────────────────────────────────── locals ───────────────────────────────────────────╮ │
│ │                                      _ = 0                                                   │ │
│ │                       add_layer_kwargs = {}                                                  │ │
│ │                  allow_multiple_layers = True                                                │ │
│ │                                   args = [                                                   │ │
│ │                                          │   (0, 32),                                        │ │
│ │                                          │   (32, 64),                                       │ │
│ │                                          │   (64, 96),                                       │ │
│ │                                          │   (96, 128),                                      │ │
│ │                                          │   (128, 160),                                     │ │
│ │                                          │   (160, 192),                                     │ │
│ │                                          │   (192, 224),                                     │ │
│ │                                          │   (224, 256),                                     │ │
│ │                                          │   (256, 288),                                     │ │
│ │                                          │   (288, 320),                                     │ │
│ │                                          │   ... +169                                        │ │
│ │                                          ]                                                   │ │
│ │                             batch_size = 32                                                  │ │
│ │                               category = 'color'                                             │ │
│ │                                channel = None                                                │ │
│ │                            chunk_shape = None                                                │ │
│ │                             chunk_size = None                                                │ │
│ │                       chunks_per_shard = None                                                │ │
│ │                               compress = False                                               │ │
│ │                          current_dtype = dtype('uint8')                                      │ │
│ │                            czi_channel = None                                                │ │
│ │                            data_format = <DataFormat.WKW: 'wkw'>                             │ │
│ │                            dimwise_max = <function dimwise_max at 0x2b9f25dbe040>            │ │
│ │                                  dtype = None                                                │ │
│ │                               executor = <cluster_tools.executors.multiprocessing.Multiproc… │ │
│ │                                          object at 0x2b9f25d894c0>                           │ │
│ │                            first_layer = None                                                │ │
│ │                                 flip_x = False                                               │ │
│ │                                 flip_y = False                                               │ │
│ │                                 flip_z = False                                               │ │
│ │                         func_per_chunk = functools.partial(<bound method                     │ │
│ │                                          PimsImages.copy_to_view of                          │ │
│ │                                          <webknossos.dataset._utils.pims_images.PimsImages   │ │
│ │                                          object at 0x2b9f25d75430>>,                         │ │
│ │                                          mag_view=MagView(name='1',                          │ │
│ │                                          bounding_box=BoundingBox(topleft=(0, 0, 0),         │ │
│ │                                          size=(4956, 5459, 5714))), is_segmentation=False,   │ │
│ │                                          dtype=dtype('uint8'))                               │ │
│ │                                 images = [                                                   │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2021/b… │ │
│ │                                          │   ... +5704                                       │ │
│ │                                          ]                                                   │ │
│ │                                  layer = Layer('Retina_macaque_MEA-align2-regions-tiffs',    │ │
│ │                                          dtype_per_channel=uint8, num_channels=1)            │ │
│ │                             layer_name = 'Retina_macaque_MEA-align2-regions-tiffs'           │ │
│ │                      layer_name_suffix = ''                                                  │ │
│ │                                    mag = Mag(_mag=Vec3Int(1,1,1))                            │ │
│ │                               mag_view = MagView(name='1',                                   │ │
│ │                                          bounding_box=BoundingBox(topleft=(0, 0, 0),         │ │
│ │                                          size=(4956, 5459, 5714)))                           │ │
│ │                             max_layers = 20                                                  │ │
│ │                            pims_images = <webknossos.dataset._utils.pims_images.PimsImages   │ │
│ │                                          object at 0x2b9f25d75430>                           │ │
│ │                       pims_open_kwargs = {}                                                  │ │
│ │                             PimsImages = <class                                              │ │
│ │                                          'webknossos.dataset._utils.pims_images.PimsImages'> │ │
│ │                        possible_layers = None                                                │ │
│ │                                   self = Dataset(PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-pro… │ │
│ │ suffix_with_pims_open_kwargs_per_layer = {'': {}}                                            │ │
│ │                                swap_xy = False                                               │ │
│ │                              timepoint = None                                                │ │
│ │                                topleft = Vec3Int(0,0,0)                                      │ │
│ │                   truncate_rgba_to_rgb = True                                                │ │
│ │                         use_bioformats = None                                                │ │
│ │                                  z_end = 5714                                                │ │
│ │                                z_start = 5696                                                │ │
│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new/lib/python3.9/site-packages/webknossos/utils. │
│ py:141 in wait_and_ensure_success                                                                │
│                                                                                                  │
│   138 │   │   │   │   if (exc := fut.exception()) is not None:                                   │
│   139 │   │   │   │   │   raise Exception(                                                       │
│   140 │   │   │   │   │   │   "".join(                                                           │
│ ❱ 141 │   │   │   │   │   │   │   [element for element in traceback.format_exception(exc)]       │
│   142 │   │   │   │   │   │   )                                                                  │
│   143 │   │   │   │   │   ) from exc                                                             │
│   144 │   │   │   │   results.append(fut.result())                                               │
│                                                                                                  │
│ ╭───────────────────────────────────── locals ─────────────────────────────────────╮             │
│ │           exc = JSONDecodeError('Expecting value: line 1 column 1 (char 0)')     │             │
│ │           fut = <Future at 0x2b9f3e399040 state=finished raised JSONDecodeError> │             │
│ │       futures = [                                                                │             │
│ │                 │   <Future at 0x2b9f3e3fae20 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b9f361fc430 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b9f361fc880 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b9f361fca30 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b9f3e3fa760 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b9f3e3faa90 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b9f3e3fa880 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b9f3e4a6d30 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b9f3e4a6730 state=finished returned tuple>,    │             │
│ │                 │   <Future at 0x2b9f3e4a6b80 state=finished returned tuple>,    │             │
│ │                 │   ... +169                                                     │             │
│ │                 ]                                                                │             │
│ │      progress = <rich.progress.Progress object at 0x2b9f3e343af0>                │             │
│ │ progress_desc = 'Creating layer from images'                                     │             │
│ │       results = []                                                               │             │
│ │          task = 0                                                                │             │
│ ╰──────────────────────────────────────────────────────────────────────────────────╯             │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
TypeError: format_exception() missing 2 required positional arguments: 'value' and 'tb'
```

Hi @elhuhdron, I added a `--pad` flag to the `convert` command in the branch: [Fix_parallel_BufferedSliceWriter_bug](https://github.com/scalableminds/webknossos-libs/tree/Fix_parallel_BufferedSliceWriter_bug). Could you try if it works when you add this flag?

To install the cuber with all dependencies, it should also be possible to run `poetry install --all-extras` in the `webknossos` directory. 

Thanks!

Hi @markbader,
> To install the cuber with all dependencies, it should also be possible to run `poetry install --all-extras` in the webknossos directory.

Thanks, this worked for me.

> I added a `--pad` flag to the convert command 
> Could you try if it works when you add this flag?

Yes, I pulled to the most recent commit on this branch and it works now. Sorry if that was the original cause. Because the error was not pointing at this and the option was removed, I thought maybe it was happening automatically now if the input images are not the same size.

I'll continue testing the new version on this branch with my other use cases for the cuber, but please merge this branch into the next release, as I still find the `--pad` option very useful. Thank you!

Hi @markbader ,
I tried verifying this change in the latest release `0.13.6` and the `--pad` option is still not exposed. Was support for this flag intended to be merged into that release?
Hi @elhuhdron,

the flag was not added but the functionality was updated. If the images differ in their shape, they are padded automatically now. 

Can you check if the automatic padding works for you? Otherwise, it would be great if you report further issues here.
Sure, no problem. I verified that this dataset still works using the branch that you had provided me before, along with adding the `--pad` flag. The new version (without `--pad`) resulted in this error:
```
ERROR:root:A future crashed with an exception: No intersection between bounding boxes BoundingBox(topleft=(0, 0, 0), size=(3590, 3861, 10469)) and BoundingBox(topleft=(0, 4096, 9216), size=(1024, 1024, 1024)).. Future: <Future at 0x2ae9cf24df10 state=finished raised AssertionError>
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new2/lib/python3.9/site-packages/webknossos/cli/c │
│ onvert.py:102 in main                                                                            │
│                                                                                                  │
│    99 │   )                                                                                      │
│   100 │                                                                                          │
│   101 │   with get_executor_for_args(args=executor_args) as executor:                            │
│ ❱ 102 │   │   Dataset.from_images(                                                               │
│   103 │   │   │   source,                                                                        │
│   104 │   │   │   target,                                                                        │
│   105 │   │   │   voxel_size,                                                                    │
│                                                                                                  │
│ ╭─────────────────────────────────────────── locals ───────────────────────────────────────────╮ │
│ │              category = None                                                                 │ │
│ │              compress = True                                                                 │ │
│ │           data_format = <DataFormat.WKW: 'wkw'>                                              │ │
│ │ distribution_strategy = <DistributionStrategy.MULTIPROCESSING: 'multiprocessing'>            │ │
│ │              executor = <cluster_tools.executors.multiprocessing_.MultiprocessingExecutor    │ │
│ │                         object at 0x2ae8b1bab460>                                            │ │
│ │         executor_args = Namespace(jobs=48, distribution_strategy='multiprocessing',          │ │
│ │                         job_resources=None)                                                  │ │
│ │         job_resources = None                                                                 │ │
│ │                  jobs = 48                                                                   │ │
│ │                  name = 'Robin_Retina_Fovea-regions'                                         │ │
│ │                source = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_Fovea… │ │
│ │                target = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_Fovea… │ │
│ │            voxel_size = VoxelSize(x=256.0, y=256.0, z=35.0)                                  │ │
│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new2/lib/python3.9/site-packages/webknossos/datas │
│ et/dataset.py:641 in from_images                                                                 │
│                                                                                                  │
│    638 │   │   │   │   )                                                                         │
│    639 │   │   │   else:                                                                         │
│    640 │   │   │   │   category = layer_category                                                 │
│ ❱  641 │   │   │   ds.add_layer_from_images(                                                     │
│    642 │   │   │   │   filepaths[0] if len(filepaths) == 1 else filepaths,                       │
│    643 │   │   │   │   layer_name,                                                               │
│    644 │   │   │   │   category=category,                                                        │
│                                                                                                  │
│ ╭─────────────────────────────────────────── locals ───────────────────────────────────────────╮ │
│ │                 batch_size = None                                                            │ │
│ │                   category = 'color'                                                         │ │
│ │                chunk_shape = None                                                            │ │
│ │           chunks_per_shard = None                                                            │ │
│ │                        cls = <class 'webknossos.dataset.dataset.Dataset'>                    │ │
│ │                   compress = True                                                            │ │
│ │                data_format = <DataFormat.WKW: 'wkw'>                                         │ │
│ │                         ds = Dataset(PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin… │ │
│ │                   executor = <cluster_tools.executors.multiprocessing_.MultiprocessingExecu… │ │
│ │                              object at 0x2ae8b1bab460>                                       │ │
│ │                  filepaths = [                                                               │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │   ... +10459                                                  │ │
│ │                              ]                                                               │ │
│ │        filepaths_per_layer = {                                                               │ │
│ │                              │   'Robin_Retina_Fovea-regions-tiffs': [                       │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │   │                                                           │ │
│ │                              PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                              │   │   ... +10459                                              │ │
│ │                              │   ]                                                           │ │
│ │                              }                                                               │ │
│ │                     flip_x = False                                                           │ │
│ │                     flip_y = False                                                           │ │
│ │                     flip_z = False                                                           │ │
│ │    get_valid_pims_suffixes = <function get_valid_pims_suffixes at 0x2ae91326f8b0>            │ │
│ │                 input_file = PosixPath('wafer05_manifest01435_1436_S1436R1436_stitched.tiff… │ │
│ │                input_files = [                                                               │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer03_manifest00684_0685_S685R685_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer02_manifest00371_0372_S372R372_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer04_manifest00360_0361_S361R361_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer01_manifest01034_1035_S1035R1035_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer01_manifest01487_1488_S1488R1488_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer03_manifest01859_1860_S1860R1860_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer03_manifest00411_0412_S412R412_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer01_manifest01511_1512_S1512R1512_stitched.tiff… │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer04_manifest00712_0713_S713R713_stitched.tiff'), │ │
│ │                              │                                                               │ │
│ │                              PosixPath('wafer05_manifest01821_1822_S1822R1822_stitched.tiff… │ │
│ │                              │   ... +10459                                                  │ │
│ │                              ]                                                               │ │
│ │                 input_path = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                input_upath = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │             layer_category = None                                                            │ │
│ │                 layer_name = 'Robin_Retina_Fovea-regions-tiffs'                              │ │
│ │ map_filepath_to_layer_name = <function                                                       │ │
│ │                              Dataset.ConversionLayerMapping._to_callable.<locals>.<lambda>   │ │
│ │                              at 0x2ae91326faf0>                                              │ │
│ │                 max_layers = 20                                                              │ │
│ │                       name = 'Robin_Retina_Fovea-regions'                                    │ │
│ │                output_path = PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/Robin_Retina_… │ │
│ │                    swap_xy = False                                                           │ │
│ │             use_bioformats = None                                                            │ │
│ │             valid_suffixes = {                                                               │ │
│ │                              │   'iff',                                                      │ │
│ │                              │   'ras',                                                      │ │
│ │                              │   'swf',                                                      │ │
│ │                              │   'gif',                                                      │ │
│ │                              │   'mov',                                                      │ │
│ │                              │   'ffmpeg',                                                   │ │
│ │                              │   'dm4',                                                      │ │
│ │                              │   'ics',                                                      │ │
│ │                              │   'png',                                                      │ │
│ │                              │   'pict',                                                     │ │
│ │                              │   ... +59                                                     │ │
│ │                              }                                                               │ │
│ │                 voxel_size = VoxelSize(x=256.0, y=256.0, z=35.0)                             │ │
│ │          z_slices_sort_key = functools.partial(<function natsort_key at 0x2ae9c82f6c10>,     │ │
│ │                              key=None, string_func=<function                                 │ │
│ │                              parse_string_factory.<locals>.func at 0x2aea2fa91430>,          │ │
│ │                              bytes_func=<function parse_bytes_factory.<locals>.<lambda> at   │ │
│ │                              0x2aea2fa914c0>, num_func=<function                             │ │
│ │                              parse_number_factory.<locals>.func at 0x2aea2fa91550>)          │ │
│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new2/lib/python3.9/site-packages/webknossos/datas │
│ et/dataset.py:1233 in add_layer_from_images                                                      │
│                                                                                                  │
│   1230 │   │   │   │   │   module="webknossos",                                                  │
│   1231 │   │   │   │   )                                                                         │
│   1232 │   │   │   │   with get_executor_for_args(None, executor) as executor:                   │
│ ❱ 1233 │   │   │   │   │   shapes_and_max_ids = wait_and_ensure_success(                         │
│   1234 │   │   │   │   │   │   executor.map_to_futures(func_per_chunk, args),                    │
│   1235 │   │   │   │   │   │   progress_desc="Creating layer from images",                       │
│   1236 │   │   │   │   │   )                                                                     │
│                                                                                                  │
│ ╭─────────────────────────────────────────── locals ───────────────────────────────────────────╮ │
│ │                                      _ = 0                                                   │ │
│ │                       add_layer_kwargs = {}                                                  │ │
│ │                  allow_multiple_layers = True                                                │ │
│ │                                   args = [                                                   │ │
│ │                                          │   (0, 1024),                                      │ │
│ │                                          │   (1024, 2048),                                   │ │
│ │                                          │   (2048, 3072),                                   │ │
│ │                                          │   (3072, 4096),                                   │ │
│ │                                          │   (4096, 5120),                                   │ │
│ │                                          │   (5120, 6144),                                   │ │
│ │                                          │   (6144, 7168),                                   │ │
│ │                                          │   (7168, 8192),                                   │ │
│ │                                          │   (8192, 9216),                                   │ │
│ │                                          │   (9216, 10240),                                  │ │
│ │                                          │   ... +1                                          │ │
│ │                                          ]                                                   │ │
│ │                             batch_size = 1024                                                │ │
│ │                               category = 'color'                                             │ │
│ │                                channel = None                                                │ │
│ │                            chunk_shape = None                                                │ │
│ │                             chunk_size = None                                                │ │
│ │                       chunks_per_shard = None                                                │ │
│ │                               compress = True                                                │ │
│ │                          current_dtype = dtype('uint8')                                      │ │
│ │                            czi_channel = None                                                │ │
│ │                            data_format = <DataFormat.WKW: 'wkw'>                             │ │
│ │                            dimwise_max = <function dimwise_max at 0x2ae8b1bd1310>            │ │
│ │                                  dtype = None                                                │ │
│ │                               executor = <cluster_tools.executors.multiprocessing_.Multipro… │ │
│ │                                          object at 0x2ae8b1bab460>                           │ │
│ │                            first_layer = None                                                │ │
│ │                                 flip_x = False                                               │ │
│ │                                 flip_y = False                                               │ │
│ │                                 flip_z = False                                               │ │
│ │                         func_per_chunk = functools.partial(<bound method                     │ │
│ │                                          PimsImages.copy_to_view of                          │ │
│ │                                          <webknossos.dataset._utils.pims_images.PimsImages   │ │
│ │                                          object at 0x2ae8b1b9de80>>,                         │ │
│ │                                          mag_view=MagView(name='1',                          │ │
│ │                                          bounding_box=BoundingBox(topleft=(0, 0, 0),         │ │
│ │                                          size=(3590, 3861, 10469))), is_segmentation=False,  │ │
│ │                                          dtype=dtype('uint8'))                               │ │
│ │                                 images = [                                                   │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/R… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/R… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/R… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/R… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/R… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/R… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/R… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/R… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/R… │ │
│ │                                          │                                                   │ │
│ │                                          PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-proc/2023/R… │ │
│ │                                          │   ... +10459                                      │ │
│ │                                          ]                                                   │ │
│ │                                  layer = Layer('Robin_Retina_Fovea-regions-tiffs',           │ │
│ │                                          dtype_per_channel=uint8, num_channels=1)            │ │
│ │                             layer_name = 'Robin_Retina_Fovea-regions-tiffs'                  │ │
│ │                      layer_name_suffix = ''                                                  │ │
│ │                                    mag = Mag(_mag=Vec3Int(1,1,1))                            │ │
│ │                               mag_view = MagView(name='1',                                   │ │
│ │                                          bounding_box=BoundingBox(topleft=(0, 0, 0),         │ │
│ │                                          size=(3590, 3861, 10469)))                          │ │
│ │                             max_layers = 20                                                  │ │
│ │                            pims_images = <webknossos.dataset._utils.pims_images.PimsImages   │ │
│ │                                          object at 0x2ae8b1b9de80>                           │ │
│ │                       pims_open_kwargs = {}                                                  │ │
│ │                             PimsImages = <class                                              │ │
│ │                                          'webknossos.dataset._utils.pims_images.PimsImages'> │ │
│ │                        possible_layers = None                                                │ │
│ │                                   self = Dataset(PosixPath('/gpfs/soma_fs/cne-mSEM/mSEM-pro… │ │
│ │ suffix_with_pims_open_kwargs_per_layer = {'': {}}                                            │ │
│ │                                swap_xy = False                                               │ │
│ │                              timepoint = None                                                │ │
│ │                                topleft = Vec3Int(0,0,0)                                      │ │
│ │                   truncate_rgba_to_rgb = True                                                │ │
│ │                         use_bioformats = None                                                │ │
│ │                                  z_end = 10469                                               │ │
│ │                                z_start = 10240                                               │ │
│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new2/lib/python3.9/site-packages/webknossos/utils │
│ .py:137 in wait_and_ensure_success                                                               │
│                                                                                                  │
│   134 │   │   with get_rich_progress() as progress:                                              │
│   135 │   │   │   task = progress.add_task(progress_desc, total=len(futures))                    │
│   136 │   │   │   for fut in as_completed(futures):                                              │
│ ❱ 137 │   │   │   │   results.append(fut.result())                                               │
│   138 │   │   │   │   progress.update(task, advance=1)                                           │
│   139 │   return results                                                                         │
│   140                                                                                            │
│                                                                                                  │
│ ╭─────────────────────────────────────── locals ───────────────────────────────────────╮         │
│ │           fut = <Future at 0x2ae913387a60 state=finished raised AssertionError>      │         │
│ │       futures = [                                                                    │         │
│ │                 │   <Future at 0x2ae913387520 state=finished raised AssertionError>, │         │
│ │                 │   <Future at 0x2ae913387a60 state=finished raised AssertionError>, │         │
│ │                 │   <Future at 0x2ae9133940d0 state=finished raised AssertionError>, │         │
│ │                 │   <Future at 0x2ae913387af0 state=finished raised AssertionError>, │         │
│ │                 │   <Future at 0x2ae9cf24d4f0 state=finished raised AssertionError>, │         │
│ │                 │   <Future at 0x2ae9cf24d100 state=finished raised AssertionError>, │         │
│ │                 │   <Future at 0x2ae9cf24d970 state=finished raised AssertionError>, │         │
│ │                 │   <Future at 0x2ae9cf24d610 state=finished raised AssertionError>, │         │
│ │                 │   <Future at 0x2ae9cf24d490 state=finished raised AssertionError>, │         │
│ │                 │   <Future at 0x2ae9cf24df10 state=finished raised AssertionError>, │         │
│ │                 │   ... +1                                                           │         │
│ │                 ]                                                                    │         │
│ │      progress = <rich.progress.Progress object at 0x2ae913272be0>                    │         │
│ │ progress_desc = 'Creating layer from images'                                         │         │
│ │       results = []                                                                   │         │
│ │          task = 0                                                                    │         │
│ ╰──────────────────────────────────────────────────────────────────────────────────────╯         │
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new2/lib/python3.9/concurrent/futures/_base.py:43 │
│ 9 in result                                                                                      │
│                                                                                                  │
│   436 │   │   │   │   if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:                     │
│   437 │   │   │   │   │   raise CancelledError()                                                 │
│   438 │   │   │   │   elif self._state == FINISHED:                                              │
│ ❱ 439 │   │   │   │   │   return self.__get_result()                                             │
│   440 │   │   │   │                                                                              │
│   441 │   │   │   │   self._condition.wait(timeout)                                              │
│   442                                                                                            │
│                                                                                                  │
│ ╭──── locals ────╮                                                                               │
│ │    self = None │                                                                               │
│ │ timeout = None │                                                                               │
│ ╰────────────────╯                                                                               │
│                                                                                                  │
│ /gpfs/soma_fs/home/watkins/mambaforge/envs/wkw-new2/lib/python3.9/concurrent/futures/_base.py:39 │
│ 1 in __get_result                                                                                │
│                                                                                                  │
│   388 │   def __get_result(self):                                                                │
│   389 │   │   if self._exception:                                                                │
│   390 │   │   │   try:                                                                           │
│ ❱ 391 │   │   │   │   raise self._exception                                                      │
│   392 │   │   │   finally:                                                                       │
│   393 │   │   │   │   # Break a reference cycle with the exception in self._exception            │
│   394 │   │   │   │   self = None                                                                │
│                                                                                                  │
│ ╭── locals ───╮                                                                                  │
│ │ self = None │                                                                                  │
│ ╰─────────────╯                                                                                  │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
AssertionError: No intersection between bounding boxes BoundingBox(topleft=(0, 0, 0), size=(3590, 3861, 10469)) and 
BoundingBox(topleft=(0, 4096, 1024), size=(1024, 1024, 1024)).
```
Thanks for reporting this! The error is caused by writing the compressed data. If the `--compress` flag is removed, it should work right now.

I am working on a solution to resolve this issue.

Resolving patch:
diff --git a/webknossos/webknossos/dataset/view.py b/webknossos/webknossos/dataset/view.py
--- a/webknossos/webknossos/dataset/view.py
+++ b/webknossos/webknossos/dataset/view.py
@@ -277,14 +277,17 @@ def write(
 
         if self._is_compressed():
             for current_mag_bbox, chunked_data in self._prepare_compressed_write(
-                current_mag_bbox, data
+                current_mag_bbox, data, json_update_allowed
             ):
                 self._array.write(current_mag_bbox.topleft, chunked_data)
         else:
             self._array.write(current_mag_bbox.topleft, data)
 
     def _prepare_compressed_write(
-        self, current_mag_bbox: BoundingBox, data: np.ndarray
+        self,
+        current_mag_bbox: BoundingBox,
+        data: np.ndarray,
+        json_update_allowed: bool = True,
     ) -> Iterator[Tuple[BoundingBox, np.ndarray]]:
         """This method takes an arbitrary sized chunk of data with an accompanying bbox,
         divides these into chunks of shard_shape size and delegates
@@ -305,10 +308,15 @@ def _prepare_compressed_write(
                     -current_mag_bbox.topleft
                 ).to_slices()
 
-            yield self._prepare_compressed_write_chunk(chunked_bbox, data[source_slice])
+            yield self._prepare_compressed_write_chunk(
+                chunked_bbox, data[source_slice], json_update_allowed
+            )
 
     def _prepare_compressed_write_chunk(
-        self, current_mag_bbox: BoundingBox, data: np.ndarray
+        self,
+        current_mag_bbox: BoundingBox,
+        data: np.ndarray,
+        json_update_allowed: bool = True,
     ) -> Tuple[BoundingBox, np.ndarray]:
         """This method takes an arbitrary sized chunk of data with an accompanying bbox
         (ideally not larger than a shard) and enlarges that chunk to fit the shard it
@@ -321,7 +329,11 @@ def _prepare_compressed_write_chunk(
         if current_mag_bbox != aligned_bbox:
             # The data bbox should either be aligned or match the dataset's bounding box:
             current_mag_view_bbox = self.bounding_box.in_mag(self._mag)
-            if current_mag_bbox != current_mag_view_bbox.intersected_with(aligned_bbox):
+            if (
+                json_update_allowed
+                and current_mag_bbox
+                != current_mag_view_bbox.intersected_with(aligned_bbox)
+            ):
                 warnings.warn(
                     _BLOCK_ALIGNMENT_WARNING,
                 )
@@ -334,8 +346,8 @@ def _prepare_compressed_write_chunk(
             # overwrite the specified data
             aligned_data[index_slice] = data
             return aligned_bbox, aligned_data
-        else:
-            return current_mag_bbox, data
+
+        return current_mag_bbox, data
 
     def read(
         self,


Test patch:
diff --git a/webknossos/tests/dataset/test_add_layer_from_images.py b/webknossos/tests/dataset/test_add_layer_from_images.py
--- a/webknossos/tests/dataset/test_add_layer_from_images.py
+++ b/webknossos/tests/dataset/test_add_layer_from_images.py
@@ -96,11 +96,11 @@ def test_compare_tifffile(tmp_path: Path) -> None:
         (1024, 1024, 13),
     ),
     (
-        "testdata/tiff_with_different_dimensions/*",
+        "testdata/tiff_with_different_shapes/*",
         {"flip_y": True},
         "uint8",
         1,
-        (2970, 2521, 3),
+        (2970, 2521, 4),
     ),
     ("testdata/various_tiff_formats/test_CS.tif", {}, "uint8", 3, (128, 128, 320)),
     ("testdata/various_tiff_formats/test_C.tif", {}, "uint8", 1, (128, 128, 320)),
diff --git a/webknossos/tests/dataset/test_from_images.py b/webknossos/tests/dataset/test_from_images.py
--- a/webknossos/tests/dataset/test_from_images.py
+++ b/webknossos/tests/dataset/test_from_images.py
@@ -63,7 +63,7 @@ def test_multiple_multitiffs(tmp_path: Path) -> None:
 def test_no_slashes_in_layername(tmp_path: Path) -> None:
     (input_path := tmp_path / "tiff" / "subfolder" / "tifffiles").mkdir(parents=True)
     copytree(
-        TESTDATA_DIR / "tiff_with_different_dimensions",
+        TESTDATA_DIR / "tiff_with_different_shapes",
         input_path,
         dirs_exist_ok=True,
     )
diff --git a/webknossos/tests/test_cli.py b/webknossos/tests/test_cli.py
--- a/webknossos/tests/test_cli.py
+++ b/webknossos/tests/test_cli.py
@@ -202,12 +202,15 @@ def test_convert() -> None:
         assert (wkw_path / PROPERTIES_FILE_NAME).exists()
 
 
-def test_convert_with_all_params() -> None:
+@pytest.mark.parametrize(
+    "origin_path",
+    [TESTDATA_DIR / "tiff", TESTDATA_DIR / "tiff_with_different_shapes"],
+)
+def test_convert_with_all_params(origin_path: Path) -> None:
     """Tests the functionality of convert subcommand."""
 
     with tmp_cwd():
-        origin_path = TESTDATA_DIR / "tiff"
-        wkw_path = Path("wkw_from_tiff_extended")
+        wkw_path = Path(f"wkw_from_{origin_path.name}")
         with pytest.warns(UserWarning):
             result = runner.invoke(
                 app,