Repository: rasterio/rasterio

Issue:
Missing masking option(s) for rio-calc
Currently, calc expressions are evaluated on arrays read with default masking. The default is going to change from `True` to `False` in 1.0, changing the results of calc expressions. Users need an option to specify whether they want expressions evaluated in masked array mode or not.

And what do we call the non-masked array mode? "Raw" mode?



Discussion:

Resolving patch:
diff --git a/rasterio/fill.py b/rasterio/fill.py
--- a/rasterio/fill.py
+++ b/rasterio/fill.py
@@ -1,41 +1,46 @@
 import rasterio
 from rasterio._fill import _fillnodata
 
+
 def fillnodata(
         image,
         mask=None,
         max_search_distance=100.0,
         smoothing_iterations=0):
     """
-    Fill selected raster regions by interpolation from the edges.
+    Fill holes in a raster dataset by interpolation from the edges.
+
+    This algorithm will interpolate values for all designated nodata
+    pixels (marked by zeros in `mask`). For each pixel a four direction
+    conic search is done to find values to interpolate from (using
+    inverse distance weighting). Once all values are interpolated, zero
+    or more smoothing iterations (3x3 average filters on interpolated
+    pixels) are applied to smooth out artifacts.
+
+    This algorithm is generally suitable for interpolating missing
+    regions of fairly continuously varying rasters (such as elevation
+    models for instance). It is also suitable for filling small holes
+    and cracks in more irregularly varying images (like aerial photos).
+    It is generally not so great for interpolating a raster from sparse
+    point data.
 
-    This algorithm will interpolate values for all designated nodata pixels
-    (marked by zeros in `mask`). For each pixel a four direction conic search
-    is done to find values to interpolate from (using inverse distance
-    weighting). Once all values are interpolated, zero or more smoothing
-    iterations (3x3 average filters on interpolated pixels) are applied to
-    smooth out artifacts.
-    
-    This algorithm is generally suitable for interpolating missing regions of
-    fairly continuously varying rasters (such as elevation models for
-    instance). It is also suitable for filling small holes and cracks in more
-    irregularly varying images (like aerial photos). It is generally not so
-    great for interpolating a raster from sparse point data.
-    
     Parameters
     ----------
-    image : numpy ndarray or rasterio Band object
-        The source  containing nodata holes.
-    mask : numpy ndarray
+    image : numpy ndarray
+        The source containing nodata holes.
+    mask : numpy ndarray or None
         A mask band indicating which pixels to interpolate. Pixels to
-        interpolate into are indicated by the value 0. Values of 1 indicate
-        areas to use during interpolation. Must be same shape as image.
+        interpolate into are indicated by the value 0. Values > 0
+        indicate areas to use during interpolation. Must be same shape
+        as image. If `None`, a mask will be diagnosed from the source
+        data.
     max_search_distance : float, optional
-        The maxmimum number of pixels to search in all directions to find
-        values to interpolate from. The default is 100.
+        The maxmimum number of pixels to search in all directions to
+        find values to interpolate from. The default is 100.
     smoothing_iterations : integer, optional
-        The number of 3x3 smoothing filter passes to run. The default is 0.
-    
+        The number of 3x3 smoothing filter passes to run. The default is
+        0.
+
     Returns
     -------
     out : numpy ndarray
@@ -44,4 +49,5 @@ def fillnodata(
     max_search_distance = float(max_search_distance)
     smoothing_iterations = int(smoothing_iterations)
     with rasterio.drivers():
-        return _fillnodata(image, mask, max_search_distance, smoothing_iterations)
+        return _fillnodata(
+            image, mask, max_search_distance, smoothing_iterations)
diff --git a/rasterio/rio/calc.py b/rasterio/rio/calc.py
--- a/rasterio/rio/calc.py
+++ b/rasterio/rio/calc.py
@@ -52,8 +52,11 @@ def read_array(ix, subix=None, dtype=None):
                                 'int32', 'float32', 'float64']),
               default=None,
               help="Output data type (default: float64).")
+@click.option('--masked/--raw',
+              default=True,
+              help="Evaluate expressions using masked arrays")
 @click.pass_context
-def calc(ctx, command, files, name, dtype):
+def calc(ctx, command, files, name, dtype, masked):
     """A raster data calculator
 
     Evaluates an expression using input datasets and writes the result
@@ -122,7 +125,7 @@ def calc(ctx, command, files, name, dtype):
                     #
                     # possibly something to do with the instance being
                     # a masked array.
-                    ctxkwds[name or '_i%d' % (i+1)] = src.read()
+                    ctxkwds[name or '_i%d' % (i+1)] = src.read(masked=masked)
 
             # Extend snuggs.
             snuggs.func_map['read'] = read_array


Test patch:
diff --git a/tests/test_rio_calc.py b/tests/test_rio_calc.py
--- a/tests/test_rio_calc.py
+++ b/tests/test_rio_calc.py
@@ -31,6 +31,8 @@ def test_multiband_calc(tmpdir):
         assert src.meta['dtype'] == 'uint8'
         data = src.read()
         assert data.min() == 125
+        assert data.data[0][0][0] == 255
+        assert data.mask[0][0][0]
 
 
 def test_singleband_calc_byindex(tmpdir):
@@ -72,7 +74,6 @@ def test_parts_calc(tmpdir):
     runner = CliRunner()
     result = runner.invoke(calc, [
                     '(asarray (+ (read 1 1) 125) (read 1 1) (read 1 1))',
-                    '--dtype', 'uint8',
                     'tests/data/shade.tif',
                     outfile],
                 catch_exceptions=False)
@@ -92,7 +93,6 @@ def test_parts_calc_2(tmpdir):
     runner = CliRunner()
     result = runner.invoke(calc, [
                     '(+ (+ (/ (read 1 1) 3.0) (/ (read 1 2) 3.0)) (/ (read 1 3) 3.0))',
-                    '--dtype', 'uint8',
                     'tests/data/RGB.byte.tif',
                     outfile],
                 catch_exceptions=False)
@@ -109,7 +109,6 @@ def test_copy_rgb(tmpdir):
     runner = CliRunner()
     result = runner.invoke(calc, [
                     '(read 1)',
-                    '--dtype', 'uint8',
                     'tests/data/RGB.byte.tif',
                     outfile],
                 catch_exceptions=False)
@@ -125,8 +124,7 @@ def test_fillnodata(tmpdir):
     outfile = str(tmpdir.join('out.tif'))
     runner = CliRunner()
     result = runner.invoke(calc, [
-                    '(asarray (fillnodata (band 1 1)) (fillnodata (band 1 2)) (fillnodata (band 1 3)))',
-                    '--dtype', 'uint8',
+                    '(asarray (fillnodata (read 1 1)) (fillnodata (read 1 2)) (fillnodata (read 1 3)))',
                     'tests/data/RGB.byte.tif',
                     outfile],
                 catch_exceptions=False)
@@ -136,16 +134,14 @@ def test_fillnodata(tmpdir):
         assert src.count == 3
         assert src.meta['dtype'] == 'uint8'
         data = src.read()
-        assert round(data.mean(), 1) == 58.6
+        assert round(data.mean(), 1) == 60.6
 
 
 def test_fillnodata_map(tmpdir):
     outfile = str(tmpdir.join('out.tif'))
     runner = CliRunner()
     result = runner.invoke(calc, [
-#                    '(asarray (map fillnodata (bands 1)))',
-                    '(asarray (map fillnodata (read 1) (!= (read 1) 0)))',
-                    '--dtype', 'uint8',
+                    '(asarray (map fillnodata (read 1)))',
                     'tests/data/RGB.byte.tif',
                     outfile],
                 catch_exceptions=False)
@@ -154,7 +150,7 @@ def test_fillnodata_map(tmpdir):
         assert src.count == 3
         assert src.meta['dtype'] == 'uint8'
         data = src.read()
-        assert round(data.mean(), 1) == 58.6
+        assert round(data.mean(), 1) == 60.6
 
 
 def test_sieve_band(tmpdir):
@@ -162,7 +158,6 @@ def test_sieve_band(tmpdir):
     runner = CliRunner()
     result = runner.invoke(calc, [
                     '(sieve (band 1 1) 42)',
-                    '--dtype', 'uint8',
                     'tests/data/shade.tif',
                     outfile],
                 catch_exceptions=False)
@@ -177,7 +172,6 @@ def test_sieve_read(tmpdir):
     runner = CliRunner()
     result = runner.invoke(calc, [
                     "(sieve (read 1 1 'uint8') 42)",
-                    '--dtype', 'uint8',
                     'tests/data/shade.tif',
                     outfile],
                 catch_exceptions=False)