Repository: scalableminds/webknossos-libs

Issue:
Default max-mag in downsample task should depend on size of dataset
If the input dataset is only 1k**3 voxel big, there is no need to create all mags until 512. I think, we can find a sensible value for this automatically.


Discussion:

Resolving patch:
diff --git a/wkcuber/api/Layer.py b/wkcuber/api/Layer.py
--- a/wkcuber/api/Layer.py
+++ b/wkcuber/api/Layer.py
@@ -25,6 +25,11 @@
 
 from wkw import wkw
 
+from wkcuber.downsampling_utils import (
+    calculate_virtual_scale_for_target_mag,
+    calculate_default_max_mag,
+)
+
 if TYPE_CHECKING:
     from wkcuber.api.Dataset import AbstractDataset, TiffDataset
 from wkcuber.api.MagDataset import (
@@ -202,7 +207,7 @@ def _pad_existing_mags_for_downsampling(
     def downsample(
         self,
         from_mag: Mag,
-        max_mag: Mag,
+        max_mag: Optional[Mag],
         interpolation_mode: str,
         compress: bool,
         anisotropic: Optional[bool] = None,
@@ -213,10 +218,17 @@ def downsample(
         assert (
             from_mag.to_layer_name() in self.mags.keys()
         ), f"Failed to downsample data. The from_mag ({from_mag}) does not exist."
-        anisotropic = anisotropic or scale is not None
-        if anisotropic:
-            if scale is None:
+
+        if max_mag is None:
+            max_mag = calculate_default_max_mag(
+                self.dataset.properties.data_layers[self.name].get_bounding_box_size()
+            )
+
+        if anisotropic and scale is None:
+            if max_mag is None:
                 scale = self.dataset.properties.scale
+            else:
+                scale = calculate_virtual_scale_for_target_mag(max_mag)
 
         self._pad_existing_mags_for_downsampling(from_mag, max_mag, scale)
 
@@ -436,7 +448,7 @@ def _get_mag_dataset_class(self) -> Type[TiledTiffMagDataset]:
     def downsample(
         self,
         from_mag: Mag,
-        max_mag: Mag,
+        max_mag: Optional[Mag],
         interpolation_mode: str,
         compress: bool,
         anisotropic: Optional[bool] = None,
diff --git a/wkcuber/downsampling.py b/wkcuber/downsampling.py
--- a/wkcuber/downsampling.py
+++ b/wkcuber/downsampling.py
@@ -1,13 +1,10 @@
-import logging
-from typing import Tuple, cast
+from typing import Optional, Tuple
 
-import numpy as np
 from argparse import ArgumentParser, Namespace
 import os
 
 from wkcuber.api.Dataset import WKDataset
 from .mag import Mag
-from .metadata import read_datasource_properties
 
 from .utils import (
     add_verbose_flag,
@@ -18,18 +15,6 @@
 )
 
 
-def calculate_virtual_scale_for_target_mag(
-    target_mag: Mag,
-) -> Tuple[float, float, float]:
-    """
-    This scale is not the actual scale of the dataset
-    The virtual scale is used for downsample_mags_anisotropic.
-    """
-    max_target_value = max(list(target_mag.to_array()))
-    scale_array = max_target_value / np.array(target_mag.to_array())
-    return cast(Tuple[float, float, float], tuple(scale_array))
-
-
 def create_parser() -> ArgumentParser:
     parser = ArgumentParser()
 
@@ -59,9 +44,11 @@ def create_parser() -> ArgumentParser:
         help="Max resolution to be downsampled. In case of anisotropic downsampling, the process is considered "
         "done when max(current_mag) >= max(max_mag) where max takes the largest dimension of the mag tuple "
         "x, y, z. For example, a maximum mag value of 8 (or 8-8-8) will stop the downsampling as soon as a "
-        "magnification is produced for which one dimension is equal or larger than 8.",
+        "magnification is produced for which one dimension is equal or larger than 8. "
+        "The default value is calculated depending on the dataset size. In the lowest Mag, the size will be "
+        "smaller than 100vx per dimension",
         type=int,
-        default=512,
+        default=None,
     )
 
     group.add_argument(
@@ -137,7 +124,7 @@ def downsample_mags_isotropic(
     path: str,
     layer_name: str,
     from_mag: Mag,
-    max_mag: Mag,
+    max_mag: Optional[Mag],
     interpolation_mode: str,
     compress: bool,
     buffer_edge_len: int = None,
@@ -150,7 +137,6 @@ def downsample_mags_isotropic(
         interpolation_mode=interpolation_mode,
         compress=compress,
         anisotropic=False,
-        scale=None,
         buffer_edge_len=buffer_edge_len,
         args=args,
     )
@@ -160,8 +146,8 @@ def downsample_mags_anisotropic(
     path: str,
     layer_name: str,
     from_mag: Mag,
-    max_mag: Mag,
-    scale: Tuple[float, float, float],
+    max_mag: Optional[Mag],
+    scale: Optional[Tuple[float, float, float]],
     interpolation_mode: str,
     compress: bool,
     buffer_edge_len: int = None,
@@ -184,42 +170,20 @@ def downsample_mags_anisotropic(
     setup_logging(args)
 
     from_mag = Mag(args.from_mag)
-    max_mag = Mag(args.max)
-    if args.anisotropic_target_mag:
-        anisotropic_target_mag = Mag(args.anisotropic_target_mag)
-
-        scale = calculate_virtual_scale_for_target_mag(anisotropic_target_mag)
+    max_mag = None if args.max is None else Mag(args.max)
 
+    if args.anisotropic_target_mag or not args.isotropic:
         downsample_mags_anisotropic(
             args.path,
             args.layer_name,
             from_mag,
-            anisotropic_target_mag,
-            scale,
-            args.interpolation_mode,
-            not args.no_compress,
-            args.buffer_cube_size,
-            args,
-        )
-    elif not args.isotropic:
-        try:
-            scale = read_datasource_properties(args.path)["scale"]
-        except Exception as exc:
-            logging.error(
-                "Could not determine scale which is necessary "
-                "to find target magnifications for anisotropic downsampling. "
-                "Does the provided dataset have a datasource-properties.json file?"
-            )
-            raise exc
-
-        downsample_mags_anisotropic(
-            args.path,
-            args.layer_name,
-            from_mag,
-            max_mag,
-            scale,
-            args.interpolation_mode,
-            not args.no_compress,
+            Mag(args.anisotropic_target_mag)
+            if args.anisotropic_target_mag
+            else max_mag,
+            scale=None,
+            interpolation_mode=args.interpolation_mode,
+            compress=not args.no_compress,
+            buffer_edge_len=args.buffer_cube_size,
             args=args,
         )
     else:
diff --git a/wkcuber/downsampling_utils.py b/wkcuber/downsampling_utils.py
--- a/wkcuber/downsampling_utils.py
+++ b/wkcuber/downsampling_utils.py
@@ -65,6 +65,27 @@ def get_next_mag(mag: Mag, scale: Optional[Tuple[float, float, float]]) -> Mag:
         )
 
 
+def calculate_virtual_scale_for_target_mag(
+    target_mag: Mag,
+) -> Tuple[float, float, float]:
+    """
+    This scale is not the actual scale of the dataset
+    The virtual scale is used for downsample_mags_anisotropic.
+    """
+    max_target_value = max(list(target_mag.to_array()))
+    scale_array = max_target_value / np.array(target_mag.to_array())
+    return cast(Tuple[float, float, float], tuple(scale_array))
+
+
+def calculate_default_max_mag(dataset_size: Tuple[int, int, int]) -> Mag:
+    # The lowest mag should have a size of ~ 100vx**2 per slice
+    max_x_y = max(dataset_size[0], dataset_size[1])
+    # highest power of 2 larger (or equal) than max_x_y divided by 100
+    # The calculated factor will be used for x, y and z here. If anisotropic downsampling takes place,
+    # the dimensions can still be downsampled independently according to the scale.
+    return Mag(max(2 ** math.ceil(math.log(max_x_y / 100, 2)), 4))  # at least 4
+
+
 def parse_interpolation_mode(
     interpolation_mode: str, layer_name: str
 ) -> InterpolationModes:


Test patch:
diff --git a/tests/test_downsampling.py b/tests/test_downsampling.py
--- a/tests/test_downsampling.py
+++ b/tests/test_downsampling.py
@@ -10,6 +10,7 @@
     downsample_cube,
     downsample_cube_job,
     get_next_mag,
+    calculate_default_max_mag,
 )
 import wkw
 from wkcuber.mag import Mag
@@ -262,3 +263,13 @@ def test_downsampling_padding() -> None:
 
         assert np.array_equal(mag1.get_view().size, expected_size)
         assert np.array_equal(mag1.get_view().global_offset, expected_offset)
+
+
+def test_default_max_mag() -> None:
+    assert calculate_default_max_mag(dataset_size=(65536, 65536, 65536)) == Mag(1024)
+    assert calculate_default_max_mag(dataset_size=(4096, 4096, 4096)) == Mag(64)
+    assert calculate_default_max_mag(dataset_size=(131072, 262144, 262144)) == Mag(4096)
+    assert calculate_default_max_mag(dataset_size=(32768, 32768, 32768)) == Mag(512)
+    assert calculate_default_max_mag(dataset_size=(16384, 65536, 65536)) == Mag(1024)
+    assert calculate_default_max_mag(dataset_size=(16384, 65536, 16384)) == Mag(1024)
+    assert calculate_default_max_mag(dataset_size=(256, 256, 256)) == Mag([4, 4, 4])