Repository: opendatacube/datacube-core

Issue:
Error reading rainfall grids
### Expected behaviour
Return an xarray Dataset like the following:

```python
<xarray.Dataset>
Dimensions:    (latitude: 1, longitude: 1, time: 366)
Coordinates:
  * time       (time) datetime64[ns] 2000-01-01 2000-01-02 2000-01-03 ...
  * latitude   (latitude) float64 -27.52
  * longitude  (longitude) float64 132.1
Data variables:
    rainfall   (time, latitude, longitude) float32 0.0 0.0 7.44684e-13 0.0 ...
Attributes:
    crs:      EPSG:4326
```

Data Cube, version 1.3.2
GDAL 2.1.3, released 2017/20/01
rasterio, version 1.0a8

And the conda environment at NCI is:
/g/data/v10/public/modules/agdc-py3-env/20170427

### Actual behaviour
Fails hard with the first file:

```python
Error opening source dataset: NETCDF:/g/data/rr5/agcd/0_05/rainfall/daily/2000/rr.2000010120000101.grid.nc:rain_day
```

And then continuing on in trying to assess the crs of the object which is None.

```python
/g/data/v10/public/modules/agdc-py3/1.5.0/lib/python3.6/site-packages/datacube/storage/storage.py in _rasterio_crs_wkt(src)
     62 if str(rasterio.__version__) >= '0.36.0':
     63     def _rasterio_crs_wkt(src):
---> 64         return str(src.crs.wkt)
     65 else:
     66     def _rasterio_crs_wkt(src):

AttributeError: 'NoneType' object has no attribute 'wkt'
```

### Steps to reproduce the behaviour
```python
import datacube
dc = datacube.Datacube()
rain = dc.load(product='bom_rainfall_grids', longitude=132.1, latitude=-27.5, time=('2000-1-1', '2001-1-1'))
```

### Environment information

* Which ``datacube --version`` are you using?
Open Data Cube core, version 1.5.0

* What datacube deployment/environment are you running against?
GDAL 2.2.1, released 2017/06/23
rasterio, version 1.0a9

The conda environment being used at NCI is:
/g/data/v10/public/modules/agdc-py3-env/20170710


Discussion:
We should read changelogs of our dependencies!

From rasterio 1.0a9:
- The `crs` property of a dataset is now `None` instead of `CRS()` when the dataset's coordinate reference system is undefined (#1057).

This will need a small code update on our part to get this working again. And we should add a test for opening datasets with overridden CRSs.

Resolving patch:
diff --git a/datacube/storage/storage.py b/datacube/storage/storage.py
--- a/datacube/storage/storage.py
+++ b/datacube/storage/storage.py
@@ -61,7 +61,10 @@ def _rasterio_resampling_method(resampling):
 
 if str(rasterio.__version__) >= '0.36.0':
     def _rasterio_crs_wkt(src):
-        return str(src.crs.wkt)
+        if src.crs:
+            return str(src.crs.wkt)
+        else:
+            return ''
 else:
     def _rasterio_crs_wkt(src):
         return str(src.crs_wkt)


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -20,6 +20,12 @@ def example_gdal_path(data_folder):
     return str(os.path.join(data_folder, 'sample_tile_151_-29.tif'))
 
 
+@pytest.fixture
+def no_crs_gdal_path(data_folder):
+    """Return the pathname of a GDAL file that doesn't contain a valid CRS."""
+    return str(os.path.join(data_folder, 'no_crs_ds.tif'))
+
+
 @pytest.fixture
 def data_folder():
     return os.path.join(os.path.split(os.path.realpath(__file__))[0], 'data')
diff --git a/tests/data/no_crs_ds.tif b/tests/data/no_crs_ds.tif
new file mode 100644
Binary files /dev/null and b/tests/data/no_crs_ds.tif differ
diff --git a/tests/storage/test_storage.py b/tests/storage/test_storage.py
--- a/tests/storage/test_storage.py
+++ b/tests/storage/test_storage.py
@@ -4,7 +4,7 @@
 
 import mock
 import netCDF4
-import numpy
+import numpy as np
 import pytest
 import rasterio.warp
 import xarray
@@ -12,7 +12,7 @@
 
 import datacube
 from datacube.model import Dataset, DatasetType, MetadataType
-from datacube.storage.storage import OverrideBandDataSource
+from datacube.storage.storage import OverrideBandDataSource, RasterFileDataSource
 from datacube.storage.storage import write_dataset_to_netcdf, reproject_and_fuse, read_from_source, Resampling, \
     DatasetSource
 from datacube.utils import geometry
@@ -30,7 +30,7 @@ def test_write_dataset_to_netcdf(tmpnetcdf_filename):
         dataset[name] = (name, coord.values, {'units': coord.units, 'crs': geobox.crs})
 
     dataset['B10'] = (geobox.dimensions,
-                      numpy.arange(10000, dtype='int16').reshape(geobox.shape),
+                      np.arange(10000, dtype='int16').reshape(geobox.shape),
                       {'nodata': 0, 'units': '1', 'crs': geobox.crs})
 
     write_dataset_to_netcdf(dataset, tmpnetcdf_filename, global_attributes={'foo': 'bar'},
@@ -57,7 +57,7 @@ def test_write_dataset_to_netcdf(tmpnetcdf_filename):
 #         dataset[name] = (name, coord.values, {'units': coord.units, 'crs': geobox.crs})
 #
 #     dataset['B10'] = (geobox.dimensions,
-#                       numpy.arange(11000, dtype='int16').reshape(geobox.shape),
+#                       np.arange(11000, dtype='int16').reshape(geobox.shape),
 #                       {'nodata': 0, 'units': '1', 'crs': geobox.crs})
 #
 #     write_dataset_to_netcdf(dataset, tmpnetcdf_filename, global_attributes={'foo': 'bar'},
@@ -70,7 +70,7 @@ def test_write_dataset_to_netcdf(tmpnetcdf_filename):
 #         assert source.transform.almost_equals(affine)
 #         assert (source.read() == dataset['B10']).all()
 #
-#         dest = numpy.empty((60, 50))
+#         dest = np.empty((60, 50))
 #         source.reproject(dest, affine, geobox.crs, 0, Resampling.nearest)
 #         assert (dest == dataset['B10'][:60, :50]).all()
 #
@@ -80,7 +80,7 @@ def test_write_dataset_to_netcdf(tmpnetcdf_filename):
 #         source.reproject(dest, affine * Affine.translation(-10, -10), geobox.crs, 0, Resampling.nearest)
 #         assert (dest[10:, 10:] == dataset['B10'][:50, :40]).all()
 #
-#         dest = numpy.empty((200, 200))
+#         dest = np.empty((200, 200))
 #         source.reproject(dest, affine, geobox.crs, 0, Resampling.nearest)
 #         assert (dest[:100, :110] == dataset['B10']).all()
 #
@@ -111,7 +111,7 @@ def test_first_source_is_priority_in_reproject_and_fuse():
     source2 = _mock_datasetsource([[2, 2], [2, 2]], crs=crs, shape=shape)
     sources = [source1, source2]
 
-    output_data = numpy.full(shape, fill_value=no_data, dtype='int16')
+    output_data = np.full(shape, fill_value=no_data, dtype='int16')
     reproject_and_fuse(sources, output_data, dst_transform=identity, dst_projection=crs, dst_nodata=no_data)
 
     assert (output_data == 1).all()
@@ -126,7 +126,7 @@ def test_second_source_used_when_first_is_empty():
     source2 = _mock_datasetsource([[2, 2], [2, 2]], crs=crs, shape=shape)
     sources = [source1, source2]
 
-    output_data = numpy.full(shape, fill_value=no_data, dtype='int16')
+    output_data = np.full(shape, fill_value=no_data, dtype='int16')
     reproject_and_fuse(sources, output_data, dst_transform=identity, dst_projection=crs, dst_nodata=no_data)
 
     assert (output_data == 2).all()
@@ -141,7 +141,7 @@ def test_mixed_result_when_first_source_partially_empty():
     source2 = _mock_datasetsource([[2, 2], [2, 2]], crs=crs, shape=shape)
     sources = [source1, source2]
 
-    output_data = numpy.full(shape, fill_value=no_data, dtype='int16')
+    output_data = np.full(shape, fill_value=no_data, dtype='int16')
     reproject_and_fuse(sources, output_data, dst_transform=identity, dst_projection=crs, dst_nodata=no_data)
 
     assert (output_data == [[1, 1], [2, 2]]).all()
@@ -154,7 +154,7 @@ def _mock_datasetsource(value, crs=None, shape=(2, 2)):
     rio_reader.crs = crs
     rio_reader.transform = identity
     rio_reader.shape = shape
-    rio_reader.read.return_value = numpy.array(value)
+    rio_reader.read.return_value = np.array(value)
 
     # Use the following if a reproject were to be required
     # def fill_array(dest, *args, **kwargs):
@@ -175,7 +175,7 @@ def test_read_from_broken_source():
     rio_reader = source1.open.return_value.__enter__.return_value
     rio_reader.read.side_effect = OSError('Read or write failed')
 
-    output_data = numpy.full(shape, fill_value=no_data, dtype='int16')
+    output_data = np.full(shape, fill_value=no_data, dtype='int16')
 
     # Check exception is raised
     with pytest.raises(OSError):
@@ -212,17 +212,17 @@ def __init__(self):
         self.nodata = -999
         self.shape = (613, 597)
 
-        self.data = numpy.full(self.shape, self.nodata, dtype='int16')
-        self.data[:512, :512] = numpy.arange(512) + numpy.arange(512).reshape((512, 1))
+        self.data = np.full(self.shape, self.nodata, dtype='int16')
+        self.data[:512, :512] = np.arange(512) + np.arange(512).reshape((512, 1))
 
     def read(self, window=None, out_shape=None):
         data = self.data
         if window:
             data = self.data[slice(*window[0]), slice(*window[1])]
         if out_shape:
-            xidx = ((numpy.arange(out_shape[1]) + 0.5) * (data.shape[1] / out_shape[1]) - 0.5).round().astype('int')
-            yidx = ((numpy.arange(out_shape[0]) + 0.5) * (data.shape[0] / out_shape[0]) - 0.5).round().astype('int')
-            data = data[numpy.meshgrid(yidx, xidx, indexing='ij')]
+            xidx = ((np.arange(out_shape[1]) + 0.5) * (data.shape[1] / out_shape[1]) - 0.5).round().astype('int')
+            yidx = ((np.arange(out_shape[0]) + 0.5) * (data.shape[0] / out_shape[0]) - 0.5).round().astype('int')
+            data = data[np.meshgrid(yidx, xidx, indexing='ij')]
         return data
 
     def reproject(self, dest, dst_transform, dst_crs, dst_nodata, resampling, **kwargs):
@@ -239,7 +239,7 @@ def reproject(self, dest, dst_transform, dst_crs, dst_nodata, resampling, **kwar
 
 
 def assert_same_read_results(source, dst_shape, dst_dtype, dst_transform, dst_nodata, dst_projection, resampling):
-    expected = numpy.empty(dst_shape, dtype=dst_dtype)
+    expected = np.empty(dst_shape, dtype=dst_dtype)
     with source.open() as src:
         rasterio.warp.reproject(src.data,
                                 expected,
@@ -251,7 +251,7 @@ def assert_same_read_results(source, dst_shape, dst_dtype, dst_transform, dst_no
                                 dst_nodata=dst_nodata,
                                 resampling=resampling)
 
-    result = numpy.empty(dst_shape, dtype=dst_dtype)
+    result = np.empty(dst_shape, dtype=dst_dtype)
     with datacube.set_options(reproject_threads=1):
         read_from_source(source,
                          result,
@@ -260,7 +260,7 @@ def assert_same_read_results(source, dst_shape, dst_dtype, dst_transform, dst_no
                          dst_projection=dst_projection,
                          resampling=resampling)
 
-    assert numpy.isclose(result, expected, atol=0, rtol=0.05, equal_nan=True).all()
+    assert np.isclose(result, expected, atol=0, rtol=0.05, equal_nan=True).all()
     return result
 
 
@@ -420,8 +420,6 @@ def fake_open():
 
 
 def test_read_raster_with_custom_crs_and_transform(example_gdal_path):
-    import numpy as np
-
     with rasterio.open(example_gdal_path) as src:
         band = rasterio.band(src, 1)
         crs = geometry.CRS('EPSG:3577')
@@ -444,6 +442,22 @@ def test_read_raster_with_custom_crs_and_transform(example_gdal_path):
         assert (dest1 == dest2).all()
 
 
+def test_read_from_file_with_missing_crs(no_crs_gdal_path):
+    """
+    We need to be able to read from data files even when GDAL can't automatically gather all the metdata.
+
+    The :class:`RasterFileDataSource` is able to override the nodata, CRS and transform attributes if necessary.
+    """
+    crs = geometry.CRS('EPSG:4326')
+    nodata = -999
+    transform = Affine(0.01, 0.0, 111.975,
+                       0.0, 0.01, -9.975)
+    data_source = RasterFileDataSource(no_crs_gdal_path, bandnumber=1, nodata=nodata, crs=crs, transform=transform)
+    with data_source.open() as src:
+        dest1 = src.read()
+        assert dest1.shape == (10, 10)
+
+
 _EXAMPLE_METADATA_TYPE = MetadataType(
     {
         'name': 'eo',