Repository: rasterio/rasterio

Issue:
Relax same-extent and same-resolution constraints in rio-merge
I'd like to allow merging of non-coincident but otherwise identically structured datasets. I don't see any blockers. We're already employing some of the required logic in the new boundless reads. Adding to the 0.17 milestone.



Discussion:
:+1: 

Friday morning status: partially working code pushed to issue242 branch, will knock this out before lunch.


Resolving patch:
diff --git a/rasterio/rio/merge.py b/rasterio/rio/merge.py
--- a/rasterio/rio/merge.py
+++ b/rasterio/rio/merge.py
@@ -1,74 +1,145 @@
 # Merge command.
 
 import logging
+import math
 import os.path
 import sys
+import warnings
 
 import click
 from cligj import files_inout_arg, format_opt
 
 import rasterio
-
 from rasterio.rio.cli import cli
+from rasterio.transform import Affine
 
 
 @cli.command(short_help="Merge a stack of raster datasets.")
 @files_inout_arg
 @format_opt
+@click.option('--bounds', nargs=4, type=float, default=None,
+              help="Output bounds: left, bottom, right, top.")
+@click.option('--res', nargs=2, type=float, default=None,
+              help="Output dataset resolution: pixel width, pixel height")
+@click.option('--nodata', '-n', type=float, default=None,
+              help="Override nodata values defined in input datasets")
 @click.pass_context
-def merge(ctx, files, driver):
+def merge(ctx, files, driver, bounds, res, nodata):
     """Copy valid pixels from input files to an output file.
 
-    All files must have the same shape, number of bands, and data type.
+    All files must have the same number of bands, data type, and
+    coordinate reference system.
+
+    Input files are merged in their listed order using the reverse
+    painter's algorithm. If the output file exists, its values will be
+    overwritten by input values.
 
-    Input files are merged in their listed order using a reverse
-    painter's algorithm.
+    Geospatial bounds and resolution of a new output file in the
+    units of the input file coordinate reference system may be provided
+    and are otherwise taken from the first input file.
     """
     import numpy as np
 
     verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 1
     logger = logging.getLogger('rio')
+
     try:
         with rasterio.drivers(CPL_DEBUG=verbosity>2):
             output = files[-1]
             files = files[:-1]
 
             with rasterio.open(files[0]) as first:
+                first_res = first.res
                 kwargs = first.meta
-                kwargs['transform'] = kwargs.pop('affine')
-                dest = np.zeros((first.count,) + first.shape, 
-                    dtype=first.dtypes[0])
+                kwargs.pop('affine')
+                nodataval = first.nodatavals[0]
+                dtype = first.dtypes[0]
 
-            nodataval = next(dest.flat)
             if os.path.exists(output):
+                # TODO: prompt user to update existing file (-i option) like:
+                # overwrite b.tif? (y/n [n]) n
+                # not overwritten
                 dst = rasterio.open(output, 'r+')
-                nodataval = dst.nodatavals[0] or nodataval
+                nodataval = dst.nodatavals[0]
+                dtype = dst.dtypes[0]
+                dest = np.zeros((dst.count,) + dst.shape, dtype=dtype)
             else:
+                # Create new output file.
+                # Extent from option or extent of all inputs.
+                if not bounds:
+                    # scan input files.
+                    xs = []
+                    ys = []
+                    for f in files:
+                        with rasterio.open(f) as src:
+                            left, bottom, right, top = src.bounds
+                            xs.extend([left, right])
+                            ys.extend([bottom, top])
+                    bounds = min(xs), min(ys), max(xs), max(ys)
+                output_transform = Affine.translation(bounds[0], bounds[3])
+
+                # Resolution/pixel size.
+                if not res:
+                    res = first_res
+                output_transform *= Affine.scale(res[0], -res[1])
+
+                # Dataset shape.
+                output_width = int(math.ceil((bounds[2]-bounds[0])/res[0]))
+                output_height = int(math.ceil((bounds[3]-bounds[1])/res[1]))
+
                 kwargs['driver'] == driver
-                dst = rasterio.open(output, 'w', **kwargs)
-                nodataval = first.nodatavals[0] or nodataval
+                kwargs['transform'] = output_transform
+                kwargs['width'] = output_width
+                kwargs['height'] = output_height
 
-            if nodataval:
-                dest.fill(nodataval)
+                dst = rasterio.open(output, 'w', **kwargs)
+                dest = np.zeros((first.count, output_height, output_width),
+                        dtype=dtype)
+
+            if nodata is not None:
+                nodataval = nodata
+
+            if nodataval is not None:
+                # Only fill if the nodataval is within dtype's range.
+                inrange = False
+                if np.dtype(dtype).kind in ('i', 'u'):
+                    info = np.iinfo(dtype)
+                    inrange = (info.min <= nodataval <= info.max)
+                elif np.dtype(dtype).kind == 'f':
+                    info = np.finfo(dtype)
+                    inrange = (info.min <= nodataval <= info.max)
+                if inrange:
+                    dest.fill(nodataval)
+                else:
+                    warnings.warn(
+                        "Input file's nodata value, %s, is beyond the valid "
+                        "range of its data type, %s. Consider overriding it "
+                        "using the --nodata option for better results." % (
+                            nodataval, dtype))
+            else:
+                nodataval = 0
 
             for fname in reversed(files):
                 with rasterio.open(fname) as src:
-                    data = src.read()
-                    try:
-                        where = np.logical_and(
-                                dest==nodataval, data.mask==False)
-                    except AttributeError:
-                        where = dest==nodataval
-                    np.copyto(dest, data, where=where)
+                    # Real World (tm) use of boundless reads.
+                    # This approach uses the maximum amount of memory to solve
+                    # the problem. Making it more efficient is a TODO.
+                    window = src.window(*dst.bounds)
+                    data = np.zeros_like(dest)
+                    data = src.read(
+                            out=data,
+                            window=window,
+                            boundless=True,
+                            masked=True)
+                    np.copyto(dest, data,
+                        where=np.logical_and(
+                        dest==nodataval, data.mask==False))
 
             if dst.mode == 'r+':
-                data = dst.read()
-                try:
-                    where = np.logical_and(
-                            dest==nodataval, data.mask==False)
-                except AttributeError:
-                    where = dest==nodataval
-                np.copyto(dest, data, where=where)
+                data = dst.read(masked=True)
+                np.copyto(dest, data,
+                    where=np.logical_and(
+                    dest==nodataval, data.mask==False))
 
             dst.write(dest)
             dst.close()


Test patch:
diff --git a/tests/test_indexing.py b/tests/test_indexing.py
--- a/tests/test_indexing.py
+++ b/tests/test_indexing.py
@@ -13,13 +13,9 @@ def test_full_window():
     with rasterio.open('tests/data/RGB.byte.tif') as src:
         assert src.window(*src.bounds) == tuple(zip((0, 0), src.shape))
 
-def test_window_exception():
+def test_window_no_exception():
     with rasterio.open('tests/data/RGB.byte.tif') as src:
         left, bottom, right, top = src.bounds
         left -= 1000.0
-        try:
-            _ = src.window(left, bottom, right, top)
-            assert False
-        except ValueError:
-            assert True
-
+        assert src.window(left, bottom, right, top) == (
+                (0, src.height), (-3, src.width))
diff --git a/tests/test_read.py b/tests/test_read.py
--- a/tests/test_read.py
+++ b/tests/test_read.py
@@ -90,19 +90,6 @@ def test_read_out_dtype_fail(self):
             except:
                 assert "failed to catch exception" is False
 
-    def test_read_out_shape_resample(self):
-        with rasterio.open('tests/data/RGB.byte.tif') as s:
-            a = numpy.zeros((7, 8), dtype=rasterio.ubyte)
-            s.read_band(1, a)
-            self.assert_(
-                repr(a) == """array([[  0,   8,   5,   7,   0,   0,   0,   0],
-       [  0,   6,  61,  15,  27,  15,  24, 128],
-       [  0,  20, 152,  23,  15,  19,  28,   0],
-       [  0,  17, 255,  25, 255,  22,  32,   0],
-       [  9,   7,  14,  16,  19,  18,  36,   0],
-       [  6,  27,  43, 207,  38,  31,  73,   0],
-       [  0,   0,   0,   0,  74,  23,   0,   0]], dtype=uint8)""", a)
-
     def test_read_basic(self):
         with rasterio.open('tests/data/shade.tif') as s:
             a = s.read()  # Gray
diff --git a/tests/test_read_boundless.py b/tests/test_read_boundless.py
--- a/tests/test_read_boundless.py
+++ b/tests/test_read_boundless.py
@@ -1,6 +1,8 @@
 import logging
 import sys
 
+import numpy
+
 import rasterio
 
 
@@ -11,6 +13,7 @@ def test_read_boundless_natural_extent():
     with rasterio.open('tests/data/RGB.byte.tif') as src:
         data = src.read(boundless=True)
         assert data.shape == (3, src.height, src.width)
+        assert abs(data[0].mean() - src.read(1).mean()) < 0.0001
         assert data.any()
 
 def test_read_boundless_beyond():
@@ -27,9 +30,22 @@ def test_read_boundless_beyond2():
         assert not data.any()
 
 
-def test_read_boundless():
+def test_read_boundless_overlap():
     with rasterio.open('tests/data/RGB.byte.tif') as src:
         data = src.read(window=((-200, 200), (-200, 200)), boundless=True)
         assert data.shape == (3, 400, 400)
         assert data.any()
         assert data[0,399,399] == 13
+
+
+def test_read_boundless_resample():
+    with rasterio.open('tests/data/RGB.byte.tif') as src:
+        out = numpy.zeros((3, 800, 800), dtype=numpy.uint8)
+        data = src.read(
+                out=out,
+                window=((-200, 200), (-200, 200)),
+                masked=True,
+                boundless=True)
+        assert data.shape == (3, 800, 800)
+        assert data.any()
+        assert data[0,798,798] == 13
diff --git a/tests/test_read_resample.py b/tests/test_read_resample.py
new file mode 100644
--- /dev/null
+++ b/tests/test_read_resample.py
@@ -0,0 +1,33 @@
+import numpy
+
+import rasterio
+
+
+# Rasterio exposes GDAL's resampling/decimation on I/O. These are the tests
+# that it does this correctly.
+#
+# Rasterio's test dataset is 718 rows by 791 columns.
+
+def test_read_out_shape_resample_down():
+    with rasterio.open('tests/data/RGB.byte.tif') as s:
+        out = numpy.zeros((7, 8), dtype=rasterio.ubyte)
+        data = s.read(1, out=out)
+        expected = numpy.array([
+            [  0,   8,   5,   7,   0,   0,   0,   0],
+            [  0,   6,  61,  15,  27,  15,  24, 128],
+            [  0,  20, 152,  23,  15,  19,  28,   0],
+            [  0,  17, 255,  25, 255,  22,  32,   0],
+            [  9,   7,  14,  16,  19,  18,  36,   0],
+            [  6,  27,  43, 207,  38,  31,  73,   0],
+            [  0,   0,   0,   0,  74,  23,   0,   0]], dtype=numpy.uint8)
+        assert (data == expected).all() # all True.
+
+
+def test_read_out_shape_resample_up():
+    # Instead of testing array items, test statistics. Upsampling by an even
+    # constant factor shouldn't change the mean.
+    with rasterio.open('tests/data/RGB.byte.tif') as s:
+        out = numpy.zeros((7180, 7910), dtype=rasterio.ubyte)
+        data = s.read(1, out=out, masked=True)
+        assert data.shape == (7180, 7910)
+        assert data.mean() == s.read(1).mean()
diff --git a/tests/test_rio_merge.py b/tests/test_rio_merge.py
--- a/tests/test_rio_merge.py
+++ b/tests/test_rio_merge.py
@@ -10,7 +10,7 @@
 from rasterio.rio.merge import merge
 
 
-logging.basicConfig(stream=sys.stderr, level=logging.INFO)
+logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
 
 
 # Fixture to create test datasets within temporary directory
@@ -18,7 +18,7 @@
 def test_data_dir_1(tmpdir):
     kwargs = {
         "crs": {'init': 'epsg:4326'},
-        "transform": (-114, 0.2, 0, 46, 0, -0.1),
+        "transform": (-114, 0.2, 0, 46, 0, -0.2),
         "count": 1,
         "dtype": rasterio.uint8,
         "driver": "GTiff",
@@ -87,6 +87,16 @@ def test_merge_with_nodata(test_data_dir_1):
         assert numpy.all(data == expected)
 
 
+def test_merge_warn(test_data_dir_1):
+    outputname = str(test_data_dir_1.join('merged.tif'))
+    inputs = [str(x) for x in test_data_dir_1.listdir()]
+    inputs.sort()
+    runner = CliRunner()
+    result = runner.invoke(merge, inputs + [outputname] + ['-n', '-1'])
+    assert result.exit_code == 0
+    assert "using the --nodata option for better results" in result.output
+
+
 def test_merge_without_nodata(test_data_dir_2):
     outputname = str(test_data_dir_2.join('merged.tif'))
     inputs = [str(x) for x in test_data_dir_2.listdir()]
@@ -144,3 +154,94 @@ def test_format_jpeg(tmpdir):
         ['tests/data/RGB.byte.tif', outputname, '--format', 'JPEG'])
     assert result.exit_code == 0
     assert os.path.exists(outputname)
+
+
+# Non-coincident datasets test fixture.
+# Two overlapping GeoTIFFs, one to the NW and one to the SE.
+@fixture(scope='function')
+def test_data_dir_overlapping(tmpdir):
+    kwargs = {
+        "crs": {'init': 'epsg:4326'},
+        "transform": (-114, 0.2, 0, 46, 0, -0.2),
+        "count": 1,
+        "dtype": rasterio.uint8,
+        "driver": "GTiff",
+        "width": 10,
+        "height": 10,
+        "nodata": 0
+    }
+
+    with rasterio.drivers():
+        with rasterio.open(str(tmpdir.join('nw.tif')), 'w', **kwargs) as dst:
+            data = numpy.ones((10, 10), dtype=rasterio.uint8)
+            dst.write_band(1, data)
+
+        kwargs['transform'] = (-113, 0.2, 0, 45, 0, -0.2)
+        with rasterio.open(str(tmpdir.join('se.tif')), 'w', **kwargs) as dst:
+            data = numpy.ones((10, 10), dtype=rasterio.uint8) * 2
+            dst.write_band(1, data)
+
+    return tmpdir
+
+
+def test_merge_overlapping(test_data_dir_overlapping):
+    outputname = str(test_data_dir_overlapping.join('merged.tif'))
+    inputs = [str(x) for x in test_data_dir_overlapping.listdir()]
+    inputs.sort()
+    runner = CliRunner()
+    result = runner.invoke(merge, inputs + [outputname])
+    assert result.exit_code == 0
+    assert os.path.exists(outputname)
+    with rasterio.open(outputname) as out:
+        assert out.count == 1
+        assert out.shape == (15, 15)
+        assert out.bounds == (-114, 43, -111, 46)
+        data = out.read_band(1, masked=False)
+        expected = numpy.zeros((15, 15), dtype=rasterio.uint8)
+        expected[0:10, 0:10] = 1
+        expected[5:, 5:] = 2
+        assert numpy.all(data == expected)
+
+
+# Fixture to create test datasets within temporary directory
+@fixture(scope='function')
+def test_data_dir_float(tmpdir):
+    kwargs = {
+        "crs": {'init': 'epsg:4326'},
+        "transform": (-114, 0.2, 0, 46, 0, -0.2),
+        "count": 1,
+        "dtype": rasterio.float64,
+        "driver": "GTiff",
+        "width": 10,
+        "height": 10,
+        "nodata": 0
+    }
+
+    with rasterio.drivers():
+        with rasterio.open(str(tmpdir.join('one.tif')), 'w', **kwargs) as dst:
+            data = numpy.zeros((10, 10), dtype=rasterio.float64)
+            data[0:6, 0:6] = 255
+            dst.write_band(1, data)
+
+        with rasterio.open(str(tmpdir.join('two.tif')), 'w', **kwargs) as dst:
+            data = numpy.zeros((10, 10), dtype=rasterio.float64)
+            data[4:8, 4:8] = 254
+            dst.write_band(1, data)
+    return tmpdir
+
+
+def test_merge_float(test_data_dir_float):
+    outputname = str(test_data_dir_float.join('merged.tif'))
+    inputs = [str(x) for x in test_data_dir_float.listdir()]
+    inputs.sort()
+    runner = CliRunner()
+    result = runner.invoke(merge, inputs + [outputname] + ['-n', '-1.5'])
+    assert result.exit_code == 0
+    assert os.path.exists(outputname)
+    with rasterio.open(outputname) as out:
+        assert out.count == 1
+        data = out.read_band(1, masked=False)
+        expected = numpy.ones((10, 10), dtype=rasterio.float64) * -1.5
+        expected[0:6, 0:6] = 255
+        expected[4:8, 4:8] = 254
+        assert numpy.all(data == expected)