Repository: opendatacube/datacube-core

Issue:
Data without embedded projection throws exception when reprojected on load()
**When I run**

```python
from datacube import Datacube

dc = Datacube()

query = {'crs': 'EPSG:3577',
 'time': ('1987-10-01', '1990-10-01'),
 'x': (349388.9787330463, 358497.9246628304),
 'y': (-2379960.5883129314, -2375926.544118764),
'output_crs': 'EPSG:3577', 
'resolution': (-25, 25),
'resampling': 'cubic',
 'product': 'bom_rainfall_grids'
}

dc.load(**query)
```

**Expect**
I expect to receive a reprojected DataArray.

**Instead**
I get the following error:

```Python traceback
Traceback (most recent call last):
  File "/g/data/v10/public/modules/agdc-py3-env/20170327/envs/agdc/lib/python3.5/site-packages/rasterio/windows.py", line 279, in evaluate
    r, c = window
ValueError: too many values to unpack (expected 2)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/g/data/v10/public/modules/agdc-py3-env/20170327/envs/agdc/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2881, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-8-991b42ee536c>", line 15, in <module>
    dc.load(**query)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/api/core.py", line 322, in load
    fuse_func=fuse_func, dask_chunks=dask_chunks)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/api/core.py", line 491, in load_data
    geobox, measurements, data_func)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/api/core.py", line 428, in create_storage
    data = data_func(measurement)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/api/core.py", line 484, in data_func
    skip_broken_datasets=skip_broken_datasets)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/api/core.py", line 550, in _fuse_measurement
    skip_broken_datasets=skip_broken_datasets)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/storage/storage.py", line 199, in reproject_and_fuse
    read_from_source(sources[0], destination, dst_transform, dst_nodata, dst_projection, resampling)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/storage/storage.py", line 155, in read_from_source
    NUM_THREADS=OPTIONS['reproject_threads'])
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/storage/storage.py", line 333, in reproject
    source = self.read(self.source)  # TODO: read only the part the we care about
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/storage/storage.py", line 330, in read
    return self.source.ds.read(indexes=self.source.bidx, window=window, out_shape=out_shape)
  File "rasterio/_io.pyx", line 207, in rasterio._io.DatasetReaderBase.read (rasterio/_io.c:5103)
  File "/g/data/v10/public/modules/agdc-py3-env/20170327/envs/agdc/lib/python3.5/site-packages/rasterio/windows.py", line 283, in evaluate
    raise ValueError("invalid window structure; expecting ints"
ValueError: invalid window structure; expecting ints((row_start, row_stop), (col_start, col_stop))
```

It looks like somewhere along the way the read _window_ is being replaced with a _rasterio.Band_ object.
Data without embedded projection throws exception when reprojected on load()
**When I run**

```python
from datacube import Datacube

dc = Datacube()

query = {'crs': 'EPSG:3577',
 'time': ('1987-10-01', '1990-10-01'),
 'x': (349388.9787330463, 358497.9246628304),
 'y': (-2379960.5883129314, -2375926.544118764),
'output_crs': 'EPSG:3577', 
'resolution': (-25, 25),
'resampling': 'cubic',
 'product': 'bom_rainfall_grids'
}

dc.load(**query)
```

**Expect**
I expect to receive a reprojected DataArray.

**Instead**
I get the following error:

```Python traceback
Traceback (most recent call last):
  File "/g/data/v10/public/modules/agdc-py3-env/20170327/envs/agdc/lib/python3.5/site-packages/rasterio/windows.py", line 279, in evaluate
    r, c = window
ValueError: too many values to unpack (expected 2)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/g/data/v10/public/modules/agdc-py3-env/20170327/envs/agdc/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2881, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-8-991b42ee536c>", line 15, in <module>
    dc.load(**query)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/api/core.py", line 322, in load
    fuse_func=fuse_func, dask_chunks=dask_chunks)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/api/core.py", line 491, in load_data
    geobox, measurements, data_func)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/api/core.py", line 428, in create_storage
    data = data_func(measurement)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/api/core.py", line 484, in data_func
    skip_broken_datasets=skip_broken_datasets)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/api/core.py", line 550, in _fuse_measurement
    skip_broken_datasets=skip_broken_datasets)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/storage/storage.py", line 199, in reproject_and_fuse
    read_from_source(sources[0], destination, dst_transform, dst_nodata, dst_projection, resampling)
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/storage/storage.py", line 155, in read_from_source
    NUM_THREADS=OPTIONS['reproject_threads'])
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/storage/storage.py", line 333, in reproject
    source = self.read(self.source)  # TODO: read only the part the we care about
  File "/g/data/v10/public/modules/agdc-py3/1.3.0/lib/python3.5/site-packages/datacube/storage/storage.py", line 330, in read
    return self.source.ds.read(indexes=self.source.bidx, window=window, out_shape=out_shape)
  File "rasterio/_io.pyx", line 207, in rasterio._io.DatasetReaderBase.read (rasterio/_io.c:5103)
  File "/g/data/v10/public/modules/agdc-py3-env/20170327/envs/agdc/lib/python3.5/site-packages/rasterio/windows.py", line 283, in evaluate
    raise ValueError("invalid window structure; expecting ints"
ValueError: invalid window structure; expecting ints((row_start, row_stop), (col_start, col_stop))
```

It looks like somewhere along the way the read _window_ is being replaced with a _rasterio.Band_ object.


Discussion:
That's one we don't have a test for. This seems to work though: 
```python
albers_grid = dc.load(product='dsm1sv10', x=(149.07, 149.17), y=(-35.25, -35.35),
                       output_crs='EPSG:3577', resolution=(-25,25))
```
Thanks for that test Simon. Glad to know it's not as widespread a problem as I first suspected.

Also, my bad for not including version/runtime information. I can confirm that it fails with the latest stable release and module available on the NCI.

```console
$ module load agdc-py3-prod
$ datacube --version
Data Cube, version 1.3.2
$ which datacube
/g/data/v10/public/modules/agdc-py3/1.3.2/bin/datacube
```
```python console
>>> import datacube
>>> datacube.__version__
'1.3.2'
>>> datacube.__path__
['/g/data/v10/public/modules/agdc-py3/1.3.2/lib/python3.6/site-packages/datacube']
````

It looks to only occur for reading formats which rasterio can't read the projection, and has to fall back to the product definition to determine a CRS.
In this case, the format is a NetCDF file without CF-compliant geospatial metadata.
That's one we don't have a test for. This seems to work though: 
```python
albers_grid = dc.load(product='dsm1sv10', x=(149.07, 149.17), y=(-35.25, -35.35),
                       output_crs='EPSG:3577', resolution=(-25,25))
```
Thanks for that test Simon. Glad to know it's not as widespread a problem as I first suspected.

Also, my bad for not including version/runtime information. I can confirm that it fails with the latest stable release and module available on the NCI.

```console
$ module load agdc-py3-prod
$ datacube --version
Data Cube, version 1.3.2
$ which datacube
/g/data/v10/public/modules/agdc-py3/1.3.2/bin/datacube
```
```python console
>>> import datacube
>>> datacube.__version__
'1.3.2'
>>> datacube.__path__
['/g/data/v10/public/modules/agdc-py3/1.3.2/lib/python3.6/site-packages/datacube']
````

It looks to only occur for reading formats which rasterio can't read the projection, and has to fall back to the product definition to determine a CRS.
In this case, the format is a NetCDF file without CF-compliant geospatial metadata.

Resolving patch:
diff --git a/datacube/storage/storage.py b/datacube/storage/storage.py
--- a/datacube/storage/storage.py
+++ b/datacube/storage/storage.py
@@ -210,6 +210,10 @@ def copyto_fuser(dest, src):
 
 
 class BandDataSource(object):
+    """Wrapper for a rasterio.Band object
+
+    :param source: rasterio.Band
+    """
     def __init__(self, source, nodata=None):
         self.source = source
         if nodata is None:
@@ -234,6 +238,8 @@ def shape(self):
         return self.source.shape
 
     def read(self, window=None, out_shape=None):
+        """Read data in the native format, returning a native array
+        """
         return self.source.ds.read(indexes=self.source.bidx, window=window, out_shape=out_shape)
 
     def reproject(self, dest, dst_transform, dst_crs, dst_nodata, resampling, **kwargs):
@@ -312,6 +318,12 @@ def reproject(self, dest, dst_transform, dst_crs, dst_nodata, resampling, **kwar
 
 
 class OverrideBandDataSource(object):
+    """Wrapper for a rasterio.Band object that overrides nodata, crs and transform
+
+    This is useful for files with malformed or missing properties
+
+    :param source: rasterio.Band
+    """
     def __init__(self, source, nodata, crs, transform):
         self.source = source
         self.nodata = nodata
@@ -327,10 +339,12 @@ def shape(self):
         return self.source.shape
 
     def read(self, window=None, out_shape=None):
+        """Read data in the native format, returning a native array
+        """
         return self.source.ds.read(indexes=self.source.bidx, window=window, out_shape=out_shape)
 
     def reproject(self, dest, dst_transform, dst_crs, dst_nodata, resampling, **kwargs):
-        source = self.read(self.source)  # TODO: read only the part the we care about
+        source = self.read()  # TODO: read only the part the we care about
         return rasterio.warp.reproject(source,
                                        dest,
                                        src_transform=self.transform,


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -11,17 +11,17 @@
 
 
 @pytest.fixture
-def example_gdal_path(request):
+def example_gdal_path(data_folder):
     """Return the pathname of a sample geotiff file
 
     Use this fixture by specifiying an argument named 'example_gdal_path' in your
     test method.
     """
-    return str(request.fspath.dirpath('data/sample_tile_151_-29.tif'))
+    return str(os.path.join(data_folder, 'sample_tile_151_-29.tif'))
 
 
 @pytest.fixture
-def data_folder(request):
+def data_folder():
     return os.path.join(os.path.split(os.path.realpath(__file__))[0], 'data')
 
 
diff --git a/tests/storage/test_storage.py b/tests/storage/test_storage.py
--- a/tests/storage/test_storage.py
+++ b/tests/storage/test_storage.py
@@ -13,7 +13,7 @@
 import datacube
 from datacube.utils import geometry
 from datacube.storage.storage import write_dataset_to_netcdf, reproject_and_fuse, read_from_source, Resampling
-from datacube.storage.storage import NetCDFDataSource
+from datacube.storage.storage import NetCDFDataSource, OverrideBandDataSource
 
 GEO_PROJ = 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],' \
            'AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],' \
@@ -399,3 +399,28 @@ def fake_open():
                  resampling=Resampling.cubic)
 
     # TODO: crs change
+
+
+def test_read_raster_with_custom_crs_and_transform(example_gdal_path):
+    import numpy as np
+
+    with rasterio.open(example_gdal_path) as src:
+        band = rasterio.band(src, 1)
+        crs = geometry.CRS('EPSG:3577')
+        nodata = -999
+        transform = Affine(25.0, 0.0, 1000000.0,
+                           0.0, -25.0, -900000.0)
+
+        # Read all raw data from source file
+        band_data_source = OverrideBandDataSource(band, nodata, crs, transform)
+        dest1 = band_data_source.read()
+        assert dest1.shape
+
+        # Attempt to read with the same transform parameters
+        dest2 = np.full(shape=(4000, 4000), fill_value=nodata, dtype=np.float32)
+        dst_transform = transform
+        dst_crs = crs
+        dst_nodata = nodata
+        resampling = datacube.storage.storage.RESAMPLING_METHODS['nearest']
+        band_data_source.reproject(dest2, dst_transform, dst_crs, dst_nodata, resampling)
+        assert (dest1 == dest2).all()