Repository: rasterio/rasterio

Issue:
Read default of masked=None breaks for some nodatavals
Back in #71, there was some controversy over the decision to have RasterReader.read's default operation as `masked=None`, which is both the slowest and most complicated behavior, and also is _different_ from `masked=False`.

Here's one case that took a colleage and I hours to debug:  It's allowed have an image whose NoDataValue is outside the range of the dtype.  If I want to read one of these, the default `read` behavior throws an error.

```
import numpy as np
data = np.array([[0, 0, 1], [1, 2, 0]])
kwargs = {
    'driver': u'GTiff',
    'dtype': 'uint8',
    'nodata': -1.7e+308,
    'height': 3,
    'width': 3,
    'count': 1
}
with rasterio.drivers():
    with rasterio.open('foo.tif', 'w', **kwargs) as dst:
        dst.write_band(1, data.astype(rasterio.uint8))

    with rasterio.open('foo.tif') as src:
        mask = src.read_band(1)
# TypeError: Fill value -1.7e+308 overflows dtype uint8
```

Obviously it's impossible for any raster values to have been flagged as NoDataValues, so the workaround is to ignore the NoDataValue entirely by setting `masked=False` (the distinction was a bit lost at first).

We probably don't want to change the default now, but here's what we can do:
- at the very least, print a warning suggesting that you may want `masked=False`
- I prefer tweaking behavior so that `masked=None` returns a masked array if at least one of the bands has a NoDataValue that is within the `dtype` bounds. (currently it only checks `nodataval is not None`)

Finally we should clarify the docstring: "masked if any of the nodatavals are not `None`", but "nodatavals" is not defined or referenced elsewhere in the documentation.



Discussion:
@IamJeffG yes, I'm in favor of the tweak you propose above. I'm already doing something like that in https://github.com/mapbox/rasterio/blob/3182186fae71c3047c7faa23f756848b07a86f5f/rasterio/rio/merge.py#L103-L118 and that should be portable. 

Agreed on the docstring change, too. I'm writing a new doc about masks and will reference that.


Resolving patch:
diff --git a/rasterio/rio/merge.py b/rasterio/rio/merge.py
--- a/rasterio/rio/merge.py
+++ b/rasterio/rio/merge.py
@@ -92,10 +92,16 @@ def merge(ctx, files, driver, bounds, res, nodata):
                 kwargs['width'] = output_width
                 kwargs['height'] = output_height
 
+                logger.debug("Kwargs: %r", kwargs)
+                logger.debug("bounds: %r", bounds)
+                logger.debug("Res: %r", res)
+
                 dst = rasterio.open(output, 'w', **kwargs)
                 dest = np.zeros((first.count, output_height, output_width),
                         dtype=dtype)
 
+                logger.debug("In merge, dest shape: %r", dest.shape)
+
             if nodata is not None:
                 nodataval = nodata
 
@@ -119,27 +125,54 @@ def merge(ctx, files, driver, bounds, res, nodata):
             else:
                 nodataval = 0
 
+            dst_w, dst_s, dst_e, dst_n = dst.bounds
+
             for fname in reversed(files):
                 with rasterio.open(fname) as src:
                     # Real World (tm) use of boundless reads.
                     # This approach uses the maximum amount of memory to solve
                     # the problem. Making it more efficient is a TODO.
-                    window = src.window(*dst.bounds)
-                    data = np.zeros_like(dest)
-                    data = src.read(
-                            out=data,
-                            window=window,
-                            boundless=True,
+
+                    # 1. Compute spatial intersection of destination
+                    #    and source.
+                    src_w, src_s, src_e, src_n = src.bounds
+
+                    int_w = src_w if src_w > dst_w else dst_w
+                    int_s = src_s if src_s > dst_s else dst_s
+                    int_e = src_e if src_e < dst_e else dst_e
+                    int_n = src_n if src_n < dst_n else dst_n
+
+                    # 2. Compute the source window.
+                    src_window = src.window(int_w, int_s, int_e, int_n)
+
+                    # 3. Compute the destination window.
+                    dst_window = dst.window(int_w, int_s, int_e, int_n)
+
+                    # 4. Initialize temp array.
+                    temp = np.zeros(
+                            (first.count,) + tuple(b - a for a, b in dst_window),
+                            dtype=dtype)
+
+                    temp = src.read(
+                            out=temp,
+                            window=src_window,
+                            boundless=False,
                             masked=True)
-                    np.copyto(dest, data,
+
+                    # 5. Copy elements of temp into dest.
+                    roff, coff = dst.index(int_w, int_n)
+                    h, w = temp.shape[-2:]
+
+                    region = dest[:,roff:roff+h,coff:coff+w]
+                    np.copyto(region, temp,
                         where=np.logical_and(
-                        dest==nodataval, data.mask==False))
+                        region==nodataval, temp.mask==False))
 
             if dst.mode == 'r+':
-                data = dst.read(masked=True)
-                np.copyto(dest, data,
+                temp = dst.read(masked=True)
+                np.copyto(dest, temp,
                     where=np.logical_and(
-                    dest==nodataval, data.mask==False))
+                    dest==nodataval, temp.mask==False))
 
             dst.write(dest)
             dst.close()


Test patch:
diff --git a/tests/test_coords.py b/tests/test_coords.py
--- a/tests/test_coords.py
+++ b/tests/test_coords.py
@@ -1,4 +1,3 @@
-
 import rasterio
 
 def test_bounds():
@@ -17,18 +16,3 @@ def test_ul():
 def test_res():
     with rasterio.open('tests/data/RGB.byte.tif') as src:
         assert tuple(round(v, 6) for v in src.res) == (300.037927, 300.041783)
-
-def test_index():
-    with rasterio.open('tests/data/RGB.byte.tif') as src:
-        assert src.index(101985.0, 2826915.0) == (0, 0)
-        assert src.index(101985.0+400.0, 2826915.0) == (0, 1)
-        assert src.index(101985.0+400.0, 2826915.0-700.0) == (2, 1)
-
-def test_window():
-    with rasterio.open('tests/data/RGB.byte.tif') as src:
-        left, bottom, right, top = src.bounds
-        assert src.window(left, bottom, right, top) == ((0, src.height), 
-                                                        (0, src.width))
-        assert src.window(left, top-400, left+400, top) == ((0, 1), (0, 1))
-        assert src.window(left, top-500, left+500, top) == ((0, 2), (0, 2))
-
diff --git a/tests/test_indexing.py b/tests/test_indexing.py
--- a/tests/test_indexing.py
+++ b/tests/test_indexing.py
@@ -1,6 +1,6 @@
-
 import rasterio
 
+
 def test_index():
     with rasterio.open('tests/data/RGB.byte.tif') as src:
         left, bottom, right, top = src.bounds
@@ -9,13 +9,37 @@ def test_index():
         assert src.index(right, bottom) == (src.height, src.width)
         assert src.index(left, bottom) == (src.height, 0)
 
+
 def test_full_window():
     with rasterio.open('tests/data/RGB.byte.tif') as src:
-        assert src.window(*src.bounds) == tuple(zip((0, 0), src.shape))
+        left, bottom, right, top = src.bounds
+        assert src.window(left, bottom, right, top) == tuple(zip((0, 0), src.shape))
+
 
 def test_window_no_exception():
     with rasterio.open('tests/data/RGB.byte.tif') as src:
         left, bottom, right, top = src.bounds
         left -= 1000.0
+        eps = 1.0e-8
         assert src.window(left, bottom, right, top) == (
-                (0, src.height), (-3, src.width))
+                (0, src.height), (-4, src.width))
+
+
+def test_index():
+    with rasterio.open('tests/data/RGB.byte.tif') as src:
+        assert src.index(101985.0, 2826915.0) == (0, 0)
+        assert src.index(101985.0+400.0, 2826915.0) == (0, 1)
+        assert src.index(101985.0+400.0, 2826915.0-700.0) == (2, 1)
+
+
+def test_window():
+    with rasterio.open('tests/data/RGB.byte.tif') as src:
+        left, bottom, right, top = src.bounds
+        dx, dy = src.res
+        eps = 1.0e-8
+        assert src.window(left+eps, bottom+eps, right-eps, top-eps) == ((0, src.height-1), 
+                                                        (0, src.width-1))
+        assert src.index(left+400, top-400) == (1, 1)
+        assert src.index(left+dx+eps, top-dy-eps) == (1, 1)
+        assert src.window(left, top-400, left+400, top) == ((0, 1), (0, 1))
+        assert src.window(left, top-2*dy-eps, left+2*dx+eps, top) == ((0, 2), (0, 2))
diff --git a/tests/test_rio_merge.py b/tests/test_rio_merge.py
--- a/tests/test_rio_merge.py
+++ b/tests/test_rio_merge.py
@@ -8,6 +8,7 @@
 
 import rasterio
 from rasterio.rio.merge import merge
+from rasterio.transform import Affine
 
 
 logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
@@ -245,3 +246,63 @@ def test_merge_float(test_data_dir_float):
         expected[0:6, 0:6] = 255
         expected[4:8, 4:8] = 254
         assert numpy.all(data == expected)
+
+
+# Test below comes from issue #288. There was an off-by-one error in
+# pasting image data into the canvas array.
+
+@fixture(scope='function')
+def tiffs(tmpdir):
+
+    data = numpy.ones((1, 1, 1), 'uint8')
+
+    kwargs = {'count': '1',
+              'driver': 'GTiff',
+              'dtype': 'uint8',
+              'height': 1,
+              'width': 1}
+
+    kwargs['transform'] = Affine( 1, 0, 1,
+                                  0,-1, 1)
+    with rasterio.open(str(tmpdir.join('a-sw.tif')), 'w', **kwargs) as r:
+        r.write(data * 40)
+
+    kwargs['transform'] = Affine( 1, 0, 2,
+                                  0,-1, 2)
+    with rasterio.open(str(tmpdir.join('b-ct.tif')), 'w', **kwargs) as r:
+        r.write(data * 60)
+
+    kwargs['transform'] = Affine( 2, 0, 3,
+                                  0,-2, 4)
+    with rasterio.open(str(tmpdir.join('c-ne.tif')), 'w', **kwargs) as r:
+        r.write(data * 90)
+
+    kwargs['transform'] = Affine( 2, 0, 2,
+                                  0,-2, 4)
+    with rasterio.open(str(tmpdir.join('d-ne.tif')), 'w', **kwargs) as r:
+        r.write(data * 120)
+
+    return tmpdir
+
+
+def test_merge_tiny(tiffs):
+    outputname = str(tiffs.join('merged.tif'))
+    inputs = [str(x) for x in tiffs.listdir()]
+    inputs.sort()
+    runner = CliRunner()
+    result = runner.invoke(merge, inputs + [outputname])
+    assert result.exit_code == 0
+
+    # Output should be
+    #
+    # [[  0 120 120  90]
+    #  [  0 120 120  90]
+    #  [  0  60   0   0]
+    #  [ 40   0   0   0]]
+
+    with rasterio.open(outputname) as src:
+        data = src.read()
+        assert (data[0][0:2,1:3] == 120).all()
+        assert (data[0][0:2,3] == 90).all()
+        assert data[0][2][1] == 60
+        assert data[0][3][0] == 40
diff --git a/tests/test_rio_sample.py b/tests/test_rio_sample.py
--- a/tests/test_rio_sample.py
+++ b/tests/test_rio_sample.py
@@ -28,7 +28,7 @@ def test_sample_stdin():
         "[220650.0, 2719200.0]\n[220650.0, 2719200.0]",
         catch_exceptions=False)
     assert result.exit_code == 0
-    assert result.output.strip() == '[28, 29, 27]\n[28, 29, 27]'
+    assert result.output.strip() == '[18, 25, 14]\n[18, 25, 14]'
 
 
 def test_sample_arg():
@@ -38,7 +38,7 @@ def test_sample_arg():
         ['tests/data/RGB.byte.tif', "[220650.0, 2719200.0]"],
         catch_exceptions=False)
     assert result.exit_code == 0
-    assert result.output.strip() == '[28, 29, 27]'
+    assert result.output.strip() == '[18, 25, 14]'
 
 
 def test_sample_bidx():
@@ -48,7 +48,7 @@ def test_sample_bidx():
         ['tests/data/RGB.byte.tif', '--bidx', '1,2', "[220650.0, 2719200.0]"],
         catch_exceptions=False)
     assert result.exit_code == 0
-    assert result.output.strip() == '[28, 29]'
+    assert result.output.strip() == '[18, 25]'
 
 
 def test_sample_bidx2():
@@ -58,7 +58,7 @@ def test_sample_bidx2():
         ['tests/data/RGB.byte.tif', '--bidx', '1..2', "[220650.0, 2719200.0]"],
         catch_exceptions=False)
     assert result.exit_code == 0
-    assert result.output.strip() == '[28, 29]'
+    assert result.output.strip() == '[18, 25]'
 
 
 def test_sample_bidx3():
@@ -68,7 +68,7 @@ def test_sample_bidx3():
         ['tests/data/RGB.byte.tif', '--bidx', '..2', "[220650.0, 2719200.0]"],
         catch_exceptions=False)
     assert result.exit_code == 0
-    assert result.output.strip() == '[28, 29]'
+    assert result.output.strip() == '[18, 25]'
 
 
 def test_sample_bidx4():
@@ -78,4 +78,4 @@ def test_sample_bidx4():
         ['tests/data/RGB.byte.tif', '--bidx', '3', "[220650.0, 2719200.0]"],
         catch_exceptions=False)
     assert result.exit_code == 0
-    assert result.output.strip() == '[27]'
+    assert result.output.strip() == '[14]'
diff --git a/tests/test_sampling.py b/tests/test_sampling.py
--- a/tests/test_sampling.py
+++ b/tests/test_sampling.py
@@ -4,7 +4,7 @@
 def test_sampling():
     with rasterio.open('tests/data/RGB.byte.tif') as src:
         data = next(src.sample([(220650.0, 2719200.0)]))
-        assert list(data) == [28, 29, 27]
+        assert list(data) == [18, 25, 14]
 
 def test_sampling_beyond_bounds():
     with rasterio.open('tests/data/RGB.byte.tif') as src:
@@ -14,4 +14,4 @@ def test_sampling_beyond_bounds():
 def test_sampling_indexes():
     with rasterio.open('tests/data/RGB.byte.tif') as src:
         data = next(src.sample([(220650.0, 2719200.0)], indexes=[2]))
-        assert list(data) == [29]
+        assert list(data) == [25]
diff --git a/tests/test_transform.py b/tests/test_transform.py
--- a/tests/test_transform.py
+++ b/tests/test_transform.py
@@ -1,13 +1,5 @@
 import rasterio
 
-def test_window():
-    with rasterio.open('tests/data/RGB.byte.tif') as src:
-        left, bottom, right, top = src.bounds
-        assert src.window(left, bottom, right, top) == ((0, src.height),
-                                                        (0, src.width))
-        assert src.window(left, top-src.res[1], left+src.res[0], top) == (
-            (0, 1), (0, 1))
-
 
 def test_window_transform():
     with rasterio.open('tests/data/RGB.byte.tif') as src: