Repository: scikit-image/scikit-image

Issue:
filters.rank.subtract_mean underflows on uint16, otherwise gives strange output
`skimage.filters.rank.subtract_mean` does not seem to behave in the way I'd expect, nor the way it is documented. Witness:

1. Undocumented shift by 127
```python
In [3]: a = np.array([[10, 10, 10]], dtype=np.uint16)                                                                                                                                                                                                                                                                                                                       

In [4]: skimage.filters.rank.subtract_mean(a, np.ones((1, 3)))                                                                                                                                                                                                                                                                                                              
Out[4]: array([[127, 127, 127]], dtype=uint16)
```

I'd expect zero values here, but I get 127. Given operation on unsigned integers, it's perhaps reasonable that we have some way to account for negative values resulting from mean subtraction, but this seems to be undocumented and certainly not what I'd expect for `uint16` input.

2. Undocumented division by two
```python
In [5]: a = np.array([[10, 10, 40]], dtype=np.uint16)                                                                                                                                                                                                                                                                                                                       

In [6]: skimage.filters.rank.subtract_mean(a, np.ones((1, 3)))                                                                                                                                                                                                                                                                                                              
Out[6]: array([[127, 122, 134]], dtype=uint16)
```
If we add ten to the mean at the center value, leaving it unchanged, the result should change by ten. But it changes by five.

3. Underflow
```python
In [7]: a = np.array([[384, 0, 384]], dtype=np.uint16)                                                                                                                                                                                                                                                                                                                      

In [8]: skimage.filters.rank.subtract_mean(a, np.ones((1, 3)))                                                                                                                                                                                                                                                                                                              
Out[8]: array([[  223, 65535,   223]], dtype=uint16)
```
That's not what I'd expect here; this function is documented as accepting `uint16` input, but it causes underflow in surprising cases. (Note that values less than 384 don't cause underflow here.)

It's easy to see that this results from the implementation at https://github.com/scikit-image/scikit-image/blob/d0c78d7809fd21116ac386f81ea964b8350b9813/skimage/filters/rank/generic_cy.pyx#L169.

Version info:
```
3.6.8 (default, Oct  9 2019, 14:04:01) 
[GCC 5.4.0 20160609]
Linux-4.16.18-rt12-x86_64-with-Ubuntu-16.04-xenial
scikit-image version: 0.16.2
numpy version: 1.17.4
```


Discussion:
@longjon thanks for the report! All of this behaviour is indeed surprising, and imho needs fixing.

Resolving patch:
diff --git a/skimage/filters/rank/generic.py b/skimage/filters/rank/generic.py
--- a/skimage/filters/rank/generic.py
+++ b/skimage/filters/rank/generic.py
@@ -48,7 +48,6 @@
 
 """
 
-import functools
 import numpy as np
 from scipy import ndimage as ndi
 from ...util import img_as_ubyte
@@ -63,7 +62,7 @@
 
 
 def _preprocess_input(image, selem=None, out=None, mask=None, out_dtype=None,
-                  pixel_size=1):
+                      pixel_size=1):
     """Preprocess and verify input for filters.rank methods.
 
     Parameters
@@ -114,9 +113,7 @@ def _preprocess_input(image, selem=None, out=None, mask=None, out_dtype=None,
     selem = np.ascontiguousarray(img_as_ubyte(selem > 0))
     image = np.ascontiguousarray(image)
 
-    if mask is None:
-        mask = np.ones(image.shape, dtype=np.uint8)
-    else:
+    if mask is not None:
         mask = img_as_ubyte(mask)
         mask = np.ascontiguousarray(mask)
 
@@ -179,17 +176,15 @@ def _apply_scalar_per_pixel(func, image, selem, out, mask, shift_x, shift_y,
 
     """
     # preprocess and verify the input
-    image, selem, out, mask, n_bins = _preprocess_input(image,
-                                                    selem,
-                                                    out,
-                                                    mask,
-                                                    out_dtype)
+    image, selem, out, mask, n_bins = _preprocess_input(image, selem,
+                                                        out, mask,
+                                                        out_dtype)
 
     # apply cython function
     func(image, selem, shift_x=shift_x, shift_y=shift_y, mask=mask,
          out=out, n_bins=n_bins)
 
-    return out.reshape(out.shape[:2])
+    return np.squeeze(out, axis=-1)
 
 
 def _apply_vector_per_pixel(func, image, selem, out, mask, shift_x, shift_y,
@@ -231,12 +226,10 @@ def _apply_vector_per_pixel(func, image, selem, out, mask, shift_x, shift_y,
 
     """
     # preprocess and verify the input
-    image, selem, out, mask, n_bins = _preprocess_input(image,
-                                                    selem,
-                                                    out,
-                                                    mask,
-                                                    out_dtype,
-                                                    pixel_size)
+    image, selem, out, mask, n_bins = _preprocess_input(image, selem,
+                                                        out, mask,
+                                                        out_dtype,
+                                                        pixel_size)
 
     # apply cython function
     func(image, selem, shift_x=shift_x, shift_y=shift_y, mask=mask,
@@ -567,6 +560,14 @@ def subtract_mean(image, selem, out=None, mask=None, shift_x=False,
     out : 2-D array (same dtype as input image)
         Output image.
 
+    Notes
+    -----
+    Subtracting the mean value may introduce underflow. To compensate
+    this potential underflow, the obtained difference is downscaled by
+    a factor of 2 and shifted by `n_bins / 2 - 1`, the median value of
+    the local histogram (`n_bins = max(3, image.max()) +1` for 16-bits
+    images and 256 otherwise).
+
     Examples
     --------
     >>> from skimage import data


Test patch:
diff --git a/skimage/filters/rank/tests/test_rank.py b/skimage/filters/rank/tests/test_rank.py
--- a/skimage/filters/rank/tests/test_rank.py
+++ b/skimage/filters/rank/tests/test_rank.py
@@ -1,4 +1,5 @@
 import os
+import pytest
 import numpy as np
 from skimage._shared.testing import (assert_equal, assert_array_equal,
                                      assert_allclose)
@@ -10,9 +11,9 @@
 from skimage.morphology import grey, disk
 from skimage.filters import rank
 from skimage.filters.rank import __all__ as all_rank_filters
+from skimage.filters.rank import subtract_mean
 from skimage._shared._warnings import expected_warnings
-from skimage._shared.testing import test_parallel, arch32, parametrize, xfail
-from pytest import param
+from skimage._shared.testing import test_parallel, parametrize
 
 
 def test_otsu_edge_case():
@@ -41,6 +42,33 @@ def test_otsu_edge_case():
     assert result[1, 1] in [141, 172]
 
 
+@pytest.mark.parametrize("dtype", [np.uint8, np.uint16])
+def test_subtract_mean_underflow_correction(dtype):
+    # Input: [10, 10, 10]
+    selem = np.ones((1, 3))
+    arr = np.array([[10, 10, 10]], dtype=dtype)
+    result = subtract_mean(arr, selem)
+
+    if dtype == np.uint8:
+        expected_val = 127
+    else:
+        expected_val = (arr.max() + 1) // 2 - 1
+
+    assert np.all(result == expected_val)
+
+    # Input: [10, 10, 40]
+    shift = 10
+    arr[0, 2] += arr.shape[1] * shift
+    result = subtract_mean(arr, selem)
+
+    if dtype == np.uint8:
+        expected_val -= shift // 2
+    else:
+        expected_val += shift
+
+    assert result[0, 1] == expected_val
+
+
 class TestRank:
     def setup(self):
         np.random.seed(0)