Repository: scalableminds/webknossos-libs

Issue:
Downsampling omitts 3/4 of data
The downsampling tool omitts 3/4 of data, when downsampling cubes exceeding a certain x and y size. 

Zoom 1
![zoom_2_1](https://user-images.githubusercontent.com/2582395/41394962-40cc0f7e-6fac-11e8-8e89-645757985147.jpg)

Zoom 2 -> data is missing
![zoom_2_2](https://user-images.githubusercontent.com/2582395/41394965-42ad6da6-6fac-11e8-9e88-bdebb2631874.jpg)

Zoom 4 -> data is missing
![zoom_2_4](https://user-images.githubusercontent.com/2582395/41394969-47000d00-6fac-11e8-9796-69c4b2683bff.jpg)





Discussion:

Resolving patch:
diff --git a/wkcuber/compress.py b/wkcuber/compress.py
--- a/wkcuber/compress.py
+++ b/wkcuber/compress.py
@@ -44,7 +44,7 @@ def create_parser():
 
 def compress_file_job(source_path, target_path):
     try:
-        logging.info("Compressing '{}' to '{}'".format(source_path, target_path))
+        logging.debug("Compressing '{}' to '{}'".format(source_path, target_path))
         ref_time = time.time()
 
         makedirs(path.dirname(target_path), exist_ok=True)
@@ -107,5 +107,5 @@ def compress(source_path, layer_name, target_path=None, mags=None, jobs=1, verbo
 if __name__ == '__main__':
     args = create_parser().parse_args()
     compress(args.source_path, args.layer_name, args.target_path, args.mag, args.jobs, args.verbose)
+    logging.info("Old files are still present in '{0}.bak'. Please remove them when not required anymore.".format(args.source_path))
 
-    
diff --git a/wkcuber/cubing.py b/wkcuber/cubing.py
--- a/wkcuber/cubing.py
+++ b/wkcuber/cubing.py
@@ -113,6 +113,7 @@ def cubing_job(target_wkw_info, _z_slice, _source_file_slice, buffer_slices, ima
     source_files = find_source_filenames(args.source_path)
     num_x, num_y, num_z = determine_source_dims_from_images(source_files)
 
+    logging.info("Found source files: count={} size={}x{}".format(num_z, num_x, num_y))
     with ParallelExecutor(args.jobs) as pool:
         # we iterate over the z layers
         for z in range(0, num_z, BLOCK_LEN):
diff --git a/wkcuber/downsampling.py b/wkcuber/downsampling.py
--- a/wkcuber/downsampling.py
+++ b/wkcuber/downsampling.py
@@ -25,6 +25,8 @@ class InterpolationModes(Enum):
     NEAREST = 2
     BILINEAR = 3
     BICUBIC = 4
+    MAX = 5
+    MIN = 6
 
 
 def create_parser():
@@ -90,45 +92,47 @@ def downsample(source_wkw_info, target_wkw_info, source_mag, target_mag, interpo
         target_mag, source_mag))
 
     factor = int(target_mag / source_mag)
-    cube_coordinates = list(set(tuple(x // factor for x in xyz)
-                                for xyz in cube_addresses(source_wkw_info)))
-    cube_coordinates.sort()
-    logging.info("Found cubes: count={} size={} min={} max={}".format(len(
-        cube_coordinates), (CUBE_EDGE_LEN,) * 3, min(cube_coordinates), max(cube_coordinates)))
+    source_cube_addresses = cube_addresses(source_wkw_info)
+    target_cube_addresses = list(set(tuple(x // factor for x in xyz)
+                                for xyz in source_cube_addresses))
+    target_cube_addresses.sort()
+    logging.debug("Found source cubes: count={} size={} min={} max={}".format(len(
+        source_cube_addresses), (CUBE_EDGE_LEN,) * 3, min(source_cube_addresses), max(source_cube_addresses)))
+    logging.debug("Found target cubes: count={} size={} min={} max={}".format(len(
+        target_cube_addresses), (CUBE_EDGE_LEN,) * 3, min(target_cube_addresses), max(target_cube_addresses)))
 
     with ParallelExecutor(jobs) as pool:
-        for cube_x, cube_y, cube_z in cube_coordinates:
+        for target_cube_xyz in target_cube_addresses:
             pool.submit(downsample_cube_job,
                         source_wkw_info, target_wkw_info,
                         factor, interpolation_mode,
-                        cube_x, cube_y, cube_z)
+                        target_cube_xyz)
 
     logging.info("Mag {0} succesfully cubed".format(target_mag))
 
 
 def downsample_cube_job(source_wkw_info, target_wkw_info, factor, interpolation_mode,
-                        cube_x, cube_y, cube_z):
+                        target_cube_xyz):
     try:
-        logging.debug("Downsampling {},{},{}".format(
-            cube_x, cube_y, cube_z))
+        logging.debug("Downsampling {}".format(target_cube_xyz))
 
         with open_wkw(source_wkw_info) as source_wkw, open_wkw(target_wkw_info) as target_wkw:
-            source_offset = tuple(
-                a * CUBE_EDGE_LEN for a in (cube_x, cube_y, cube_z))
-            target_offset = tuple(a // factor for a in source_offset)
+            target_offset = tuple(a * CUBE_EDGE_LEN for a in target_cube_xyz)
+            source_offset = tuple(a * factor for a in target_offset)
 
             ref_time = time.time()
-            cube_buffer = source_wkw.read(source_offset, (CUBE_EDGE_LEN,) * 3)[0]
+            cube_buffer = source_wkw.read(source_offset, (CUBE_EDGE_LEN * factor,) * 3)
+            assert cube_buffer.shape[0] == 1, "Only single-channel data is supported"
+            cube_buffer = cube_buffer[0]
             if np.all(cube_buffer == 0):
-                logging.debug("Skipping empty cube {},{},{}".format(
-                    cube_x, cube_y, cube_z))
+                logging.debug("Skipping empty cube {}".format(target_cube_xyz))
             cube_data = downsample_cube(cube_buffer, factor, interpolation_mode)
             target_wkw.write(target_offset, cube_data)
 
-        logging.debug("Downsampling of {},{},{} took {:.8f}s".format(
-            cube_x, cube_y, cube_z, time.time() - ref_time))
+        logging.debug("Downsampling of {} took {:.8f}s".format(
+            target_cube_xyz, time.time() - ref_time))
     except Exception as exc:
-        logging.error("Downsampling of {},{},{} failed with {}".format(cube_x, cube_y, cube_z, exc))
+        logging.error("Downsampling of {} failed with {}".format(target_cube_xyz, exc))
         raise exc
 
 
@@ -171,6 +175,13 @@ def linear_filter_3d(data, factor, order):
                 mode='nearest',
                 prefilter=True)
 
+def _max(x):
+    return np.max(x, axis=0)
+
+
+def _min(x):
+    return np.min(x, axis=0)
+
 
 def _median(x):
     return np.median(x, axis=0).astype(x.dtype)
@@ -191,6 +202,10 @@ def downsample_cube(cube_buffer, factor, interpolation_mode):
         return linear_filter_3d(cube_buffer, factor, 1)
     elif interpolation_mode == InterpolationModes.BICUBIC:
         return linear_filter_3d(cube_buffer, factor, 2)
+    elif interpolation_mode == InterpolationModes.MAX:
+        return non_linear_filter_3d(cube_buffer, factor, _max)
+    elif interpolation_mode == InterpolationModes.MIN:
+        return non_linear_filter_3d(cube_buffer, factor, _min)
     else:
         raise Exception(
             "Invalid interpolation mode: {}".format(interpolation_mode))
@@ -221,4 +236,3 @@ def downsample_mags(path, layer_name, max_mag, dtype='uint8', interpolation_mode
     args = create_parser().parse_args()
     downsample_mags(args.path, args.layer_name, args.max, args.dtype, args.interpolation_mode, args.jobs, args.verbose)
 
-    


Test patch:
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
diff --git a/tests/test_downsampling.py b/tests/test_downsampling.py
new file mode 100644
--- /dev/null
+++ b/tests/test_downsampling.py
@@ -0,0 +1,67 @@
+import numpy as np
+from wkcuber.downsampling import (
+    CUBE_EDGE_LEN,
+    InterpolationModes,
+    downsample_cube,
+    downsample_cube_job,
+    cube_addresses,
+)
+from wkcuber.utils import WkwDatasetInfo, open_wkw
+
+WKW_CUBE_SIZE = 1024
+
+source_info = WkwDatasetInfo("testdata/WT1_wkw", "color", "uint8", 1)
+target_info = WkwDatasetInfo("testoutput/WT1_wkw", "color", "uint8", 2)
+
+
+def read_wkw(wkw_info, offset, size):
+    with open_wkw(wkw_info) as wkw_dataset:
+        return wkw_dataset.read(offset, size)[0]
+
+
+def test_downsample_cube():
+    buffer = np.zeros((CUBE_EDGE_LEN,) * 3, dtype=np.uint8)
+    buffer[:, :, :] = np.arange(0, CUBE_EDGE_LEN)
+
+    output = downsample_cube(buffer, 2, InterpolationModes.MEDIAN)
+
+    assert output.shape == (CUBE_EDGE_LEN // 2,) * 3
+    assert buffer[0, 0, 0] == 0
+    assert buffer[0, 0, 1] == 1
+    assert np.all(output[:, :, :] == np.arange(0, CUBE_EDGE_LEN, 2))
+
+
+def test_cube_addresses():
+    cubes_per_file = WKW_CUBE_SIZE // CUBE_EDGE_LEN
+
+    addresses = cube_addresses(source_info)
+    assert len(addresses) == (5 * cubes_per_file) * (5 * cubes_per_file) * (
+        1 * cubes_per_file
+    )
+    assert min(addresses) == (0, 0, 0)
+    assert max(addresses) == (
+        5 * cubes_per_file - 1,
+        5 * cubes_per_file - 1,
+        1 * cubes_per_file - 1,
+    )
+
+
+def test_downsample_cube_job():
+    offset = (15, 15, 0)
+    downsample_cube_job(source_info, target_info, 2, InterpolationModes.MAX, offset)
+
+    source_buffer = read_wkw(
+        source_info,
+        tuple(a * CUBE_EDGE_LEN * 2 for a in offset),
+        (CUBE_EDGE_LEN * 2,) * 3,
+    )
+    assert np.any(source_buffer != 0)
+
+    target_buffer = read_wkw(
+        target_info, tuple(a * CUBE_EDGE_LEN for a in offset), (CUBE_EDGE_LEN,) * 3
+    )
+    assert np.any(target_buffer != 0)
+
+    assert np.all(
+        target_buffer == downsample_cube(source_buffer, 2, InterpolationModes.MAX)
+    )