Repository: scalableminds/webknossos-libs

Issue:
Too noisy log in downsampling for small cube sizes
When using a small cube size, the logging output `Downsampling of (13, 6, 0) took 0.00587106s` is way too noisy. Consider omitting the output if the duration is < 5 seconds (?).


Discussion:

Resolving patch:
diff --git a/wkcuber/downsampling.py b/wkcuber/downsampling.py
--- a/wkcuber/downsampling.py
+++ b/wkcuber/downsampling.py
@@ -1,4 +1,6 @@
 import logging
+import math
+
 import wkw
 import re
 import numpy as np
@@ -187,7 +189,15 @@ def downsample(
 
     with get_executor_for_args(args) as executor:
         job_args = []
-        for target_cube_xyz in target_cube_addresses:
+        voxel_count_per_cube = (
+            source_wkw.header.file_len * source_wkw.header.block_len
+        ) ** 3
+        job_count_per_log = math.ceil(
+            1024 ** 3 / voxel_count_per_cube
+        )  # log every gigavoxel of processed data
+        for i, target_cube_xyz in enumerate(target_cube_addresses):
+            use_logging = i % job_count_per_log == 0
+
             job_args.append(
                 (
                     source_wkw_info,
@@ -197,6 +207,7 @@ def downsample(
                     target_cube_xyz,
                     buffer_edge_len,
                     compress,
+                    use_logging,
                 )
             )
         wait_and_ensure_success(executor.map_to_futures(downsample_cube_job, job_args))
@@ -213,12 +224,15 @@ def downsample_cube_job(args):
         target_cube_xyz,
         buffer_edge_len,
         compress,
+        use_logging,
     ) = args
 
-    logging.info("Downsampling of {}".format(target_cube_xyz))
+    if use_logging:
+        logging.info("Downsampling of {}".format(target_cube_xyz))
 
     try:
-        time_start("Downsampling of {}".format(target_cube_xyz))
+        if use_logging:
+            time_start("Downsampling of {}".format(target_cube_xyz))
         header_block_type = (
             wkw.Header.BLOCK_TYPE_LZ4HC if compress else wkw.Header.BLOCK_TYPE_RAW
         )
@@ -267,13 +281,7 @@ def downsample_cube_job(args):
                     for channel_index in range(num_channels):
                         cube_buffer = cube_buffer_channels[channel_index]
 
-                        if np.all(cube_buffer == 0):
-                            logging.debug(
-                                "        Skipping empty cube {} (tile {})".format(
-                                    target_cube_xyz, tile
-                                )
-                            )
-                        else:
+                        if not np.all(cube_buffer == 0):
                             # Downsample the buffer
 
                             data_cube = downsample_cube(
@@ -292,7 +300,8 @@ def downsample_cube_job(args):
 
                 # Write the downsampled buffer to target
                 target_wkw.write(file_offset, file_buffer)
-        time_stop("Downsampling of {}".format(target_cube_xyz))
+        if use_logging:
+            time_stop("Downsampling of {}".format(target_cube_xyz))
 
     except Exception as exc:
         logging.error("Downsampling of {} failed with {}".format(target_cube_xyz, exc))
@@ -301,7 +310,6 @@ def downsample_cube_job(args):
 
 def non_linear_filter_3d(data, factors, func):
     ds = data.shape
-    logging.info(f"{data.shape}, {factors}")
     assert not any((d % factor > 0 for (d, factor) in zip(ds, factors)))
     data = data.reshape((ds[0], factors[1], ds[1] // factors[1], ds[2]), order="F")
     data = data.swapaxes(0, 1)
@@ -652,7 +660,6 @@ def detect_larger_and_smaller_dimension(scale):
 
     from_mag = Mag(args.from_mag)
     max_mag = Mag(args.max)
-
     if args.anisotropic_target_mag:
         anisotropic_target_mag = Mag(args.anisotropic_target_mag)
 


Test patch:
diff --git a/tests/test_downsampling.py b/tests/test_downsampling.py
--- a/tests/test_downsampling.py
+++ b/tests/test_downsampling.py
@@ -103,6 +103,7 @@ def downsample_test_helper(use_compress):
         offset,
         CUBE_EDGE_LEN,
         use_compress,
+        True,
     )
     downsample_cube_job(downsample_args)
 
@@ -171,6 +172,7 @@ def test_downsample_multi_channel():
         tuple(a * WKW_CUBE_SIZE for a in offset),
         CUBE_EDGE_LEN,
         False,
+        True,
     )
     downsample_cube_job(downsample_args)