Repository: opendatacube/datacube-core

Issue:
Multiple issues with write_geotiff() method
### Expected behaviour
I should be able to write a small GeoTiff file using the `datacube.helpers.write_geotiff()` method.

I should also not have to specify a `time_index`.


### Actual behaviour

 For a 50x50 raster it fails with:
```python
    ValueError: blockxsize exceeds raster width
```

And if `time_index` is not specified it fails with:
```pytb
Exception ignored in: 'pandas._libs.lib.is_bool_array'
TypeError: len() of unsized object
Traceback (most recent call last):
  File "testlanwei.py", line 30, in <module>
    write_geotiff(filename='H8_test_1.tif', dataset=data, profile_override={'blockxsize':16, 'blockysize':16})
  File "/home/547/dra547/github/datacube-core/datacube/helpers.py", line 48, in write_geotiff
    dest.write(data.isel(time=time_index).data, bandnum)
  File "/g/data/v10/public/modules/agdc-py3-env/20170713/envs/agdc/lib/python3.6/site-packages/xarray/core/dataarray.py", line 679, in isel
    ds = self._to_temp_dataset().isel(drop=drop, **indexers)
  File "/g/data/v10/public/modules/agdc-py3-env/20170713/envs/agdc/lib/python3.6/site-packages/xarray/core/dataset.py", line 1143, in isel
    new_var = var.isel(**var_indexers)
  File "/g/data/v10/public/modules/agdc-py3-env/20170713/envs/agdc/lib/python3.6/site-packages/xarray/core/variable.py", line 570, in isel
    return self[tuple(key)]
  File "/g/data/v10/public/modules/agdc-py3-env/20170713/envs/agdc/lib/python3.6/site-packages/xarray/core/variable.py", line 1225, in __getitem__
    values = self._indexable_data[key]
  File "/g/data/v10/public/modules/agdc-py3-env/20170713/envs/agdc/lib/python3.6/site-packages/xarray/core/indexing.py", line 545, in __getitem__
    result = self.array[key]
  File "/g/data/v10/public/modules/agdc-py3-env/20170713/envs/agdc/lib/python3.6/site-packages/pandas/core/indexes/datetimelike.py", line 274, in __getitem__
    if com.is_bool_indexer(key):
  File "/g/data/v10/public/modules/agdc-py3-env/20170713/envs/agdc/lib/python3.6/site-packages/pandas/core/common.py", line 189, in is_bool_indexer
    raise ValueError('cannot index with vector containing '
ValueError: cannot index with vector containing NA / NaN values
```


The function should be changed to take more useful parameters, and should have unit test coverage. 


### Steps to reproduce the behaviour

Run the following script [broken_geotiff_write.txt](https://github.com/opendatacube/datacube-core/files/1204141/broken_geotiff_write.txt).



### Environment information

* Which ``datacube --version`` are you using?
`1.5.1`
* What datacube deployment/enviornment are you running against?
`agdc-py3-prod/1.5.1`


Discussion:

Resolving patch:
diff --git a/datacube/helpers.py b/datacube/helpers.py
--- a/datacube/helpers.py
+++ b/datacube/helpers.py
@@ -4,8 +4,9 @@
 Not used internally, those should go in `utils.py`
 """
 from __future__ import absolute_import
-import rasterio
+
 import numpy as np
+import rasterio
 
 DEFAULT_PROFILE = {
     'blockxsize': 256,
@@ -18,17 +19,22 @@
     'tiled': True}
 
 
-def write_geotiff(filename, dataset, time_index=None, profile_override=None):
+def write_geotiff(filename, dataset, profile_override=None, time_index=None):
     """
-    Write an xarray dataset to a geotiff
+    Write an ODC style xarray.Dataset to a GeoTIFF file.
 
     :param filename: Output filename
-    :attr dataset: xarray dataset containing multiple bands to write to file
-    :attr time_index: time index to write to file
-    :attr profile_override: option dict, overrides rasterio file creation options.
+    :param dataset: xarray dataset containing one or more bands to write to a file.
+    :param profile_override: option dict, overrides rasterio file creation options.
+    :param time_index: DEPRECATED
     """
     profile_override = profile_override or {}
 
+    if time_index is not None:
+        raise ValueError('''The write_geotiff function no longer supports passing in `time_index`.
+        The same function can be achieved by calling `dataset.isel(time=<time_index>)` before passing
+        in your dataset. It was removed because it made the function much less useful for more advanced cases.''')
+
     try:
         dtypes = {val.dtype for val in dataset.data_vars.values()}
         assert len(dtypes) == 1  # Check for multiple dtypes
@@ -39,19 +45,35 @@ def write_geotiff(filename, dataset, time_index=None, profile_override=None):
     profile.update({
         'width': dataset.dims[dataset.crs.dimensions[1]],
         'height': dataset.dims[dataset.crs.dimensions[0]],
-        'affine': dataset.affine,
+        'transform': dataset.affine,
         'crs': dataset.crs.crs_str,
         'count': len(dataset.data_vars),
         'dtype': str(dtypes.pop())
     })
     profile.update(profile_override)
 
+    _calculate_blocksize(profile)
+
     with rasterio.open(str(filename), 'w', **profile) as dest:
         if hasattr(dataset, 'data_vars'):
             for bandnum, data in enumerate(dataset.data_vars.values(), start=1):
-                dest.write(data.isel(time=time_index).data, bandnum)
-        else:  # Assume that we have a DataArray
-            dest.write(dataset.isel(time=time_index).data, 0)
+                dest.write(data.data, bandnum)
+
+
+def _calculate_blocksize(profile):
+    # Block size must be smaller than the image size, and for geotiffs must be divisible by 16
+    # Fix for small images.
+    if profile['blockxsize'] > profile['width']:
+        if profile['width'] % 16 == 0 or profile['width'] < 16:
+            profile['blockxsize'] = profile['width']
+        else:
+            profile['blockxsize'] = 16
+
+    if profile['blockysize'] > profile['height']:
+        if profile['height'] % 16 == 0 or profile['height'] < 16:
+            profile['blockysize'] = profile['height']
+        else:
+            profile['blockysize'] = 16
 
 
 def ga_pq_fuser(dest, src):


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -7,7 +7,13 @@
 from __future__ import print_function, absolute_import
 
 import os
+
+import numpy as np
 import pytest
+import xarray
+from affine import Affine
+
+from datacube.utils import geometry
 
 
 @pytest.fixture
@@ -28,11 +34,13 @@ def no_crs_gdal_path(data_folder):
 
 @pytest.fixture
 def data_folder():
+    """Return a string path to the location `test/data`"""
     return os.path.join(os.path.split(os.path.realpath(__file__))[0], 'data')
 
 
 @pytest.fixture
 def example_netcdf_path(request):
+    """Return a string path to `sample_tile.nc` in the test data dir"""
     return str(request.fspath.dirpath('data/sample_tile.nc'))
 
 
@@ -41,7 +49,34 @@ def example_netcdf_path(request):
 
 @pytest.fixture
 def tmpnetcdf_filename(tmpdir):
+    """Return a generated filename for a non-existant netcdf file"""
     global netcdf_num
     filename = str(tmpdir.join('testfile_np_%s.nc' % netcdf_num))
     netcdf_num += 1
     return filename
+
+
+@pytest.fixture
+def odc_style_xr_dataset():
+    """An xarray.Dataset with ODC style coordinates and CRS, and no time dimension.
+
+    Contains an EPSG:4326, single variable 'B10' of 100x100 int16 pixels."""
+    affine = Affine.scale(0.1, 0.1) * Affine.translation(20, 30)
+    geobox = geometry.GeoBox(100, 100, affine, geometry.CRS(GEO_PROJ))
+    dataset = xarray.Dataset(attrs={'extent': geobox.extent, 'crs': geobox.crs})
+    for name, coord in geobox.coordinates.items():
+        dataset[name] = (name, coord.values, {'units': coord.units, 'crs': geobox.crs})
+    dataset['B10'] = (geobox.dimensions,
+                      np.arange(10000, dtype='int16').reshape(geobox.shape),
+                      {'nodata': 0, 'units': '1', 'crs': geobox.crs})
+
+    # To include a time dimension:
+    # dataset['B10'] = (('time', 'latitude', 'longitude'), np.arange(10000, dtype='int16').reshape((1, 100, 100)),
+    #  {'nodata': 0, 'units': '1', 'crs': geobox.crs})
+
+    return dataset
+
+
+GEO_PROJ = 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],' \
+           'AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],' \
+           'AUTHORITY["EPSG","4326"]]'
diff --git a/tests/storage/test_storage.py b/tests/storage/test_storage.py
--- a/tests/storage/test_storage.py
+++ b/tests/storage/test_storage.py
@@ -7,7 +7,6 @@
 import numpy as np
 import pytest
 import rasterio.warp
-import xarray
 from affine import Affine, identity
 
 import datacube
@@ -20,30 +19,16 @@
 from datacube.utils import geometry
 from datacube.utils.geometry import GeoBox, CRS
 
-GEO_PROJ = 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],' \
-           'AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],' \
-           'AUTHORITY["EPSG","4326"]]'
 
-
-def test_write_dataset_to_netcdf(tmpnetcdf_filename):
-    affine = Affine.scale(0.1, 0.1) * Affine.translation(20, 30)
-    geobox = geometry.GeoBox(100, 100, affine, geometry.CRS(GEO_PROJ))
-    dataset = xarray.Dataset(attrs={'extent': geobox.extent, 'crs': geobox.crs})
-    for name, coord in geobox.coordinates.items():
-        dataset[name] = (name, coord.values, {'units': coord.units, 'crs': geobox.crs})
-
-    dataset['B10'] = (geobox.dimensions,
-                      np.arange(10000, dtype='int16').reshape(geobox.shape),
-                      {'nodata': 0, 'units': '1', 'crs': geobox.crs})
-
-    write_dataset_to_netcdf(dataset, tmpnetcdf_filename, global_attributes={'foo': 'bar'},
+def test_write_dataset_to_netcdf(tmpnetcdf_filename, odc_style_xr_dataset):
+    write_dataset_to_netcdf(odc_style_xr_dataset, tmpnetcdf_filename, global_attributes={'foo': 'bar'},
                             variable_params={'B10': {'attrs': {'abc': 'xyz'}}})
 
     with netCDF4.Dataset(tmpnetcdf_filename) as nco:
         nco.set_auto_mask(False)
         assert 'B10' in nco.variables
         var = nco.variables['B10']
-        assert (var[:] == dataset['B10'].values).all()
+        assert (var[:] == odc_style_xr_dataset['B10'].values).all()
 
         assert 'foo' in nco.ncattrs()
         assert nco.getncattr('foo') == 'bar'
@@ -568,6 +553,8 @@ def test_read_from_file_with_missing_crs(self, no_crs_gdal_path):
 
 @pytest.fixture
 def make_sample_netcdf(tmpdir):
+    """Make a test Geospatial NetCDF file, 4000x4000 int16 random data, in a variable named `sample`.
+    Return the GDAL access string."""
     sample_nc = str(tmpdir.mkdir('netcdfs').join('sample.nc'))
     geobox = GeoBox(4000, 4000, affine=Affine(25.0, 0.0, 1200000, 0.0, -25.0, -4200000), crs=CRS('EPSG:3577'))
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -7,11 +7,13 @@
 import string
 
 import pytest
+import rasterio
 from dateutil.parser import parse
-from pandas import to_datetime
 from hypothesis import given
 from hypothesis.strategies import integers, text
+from pandas import to_datetime
 
+from datacube.helpers import write_geotiff
 from datacube.utils import uri_to_local_path, clamp, gen_password, write_user_secret_file, slurp
 from datacube.utils.changes import check_doc_unchanged, get_doc_changes, MISSING, DocumentMismatchError
 from datacube.utils.dates import date_sequence
@@ -157,3 +159,26 @@ def test_more_check_doc_unchanged():
 
     with pytest.raises(DocumentMismatchError, message='Letters differs from stored (a.b: 1!=2)'):
         check_doc_unchanged({'a': {'b': 1}}, {'a': {'b': 2}}, 'Letters')
+
+
+def test_write_geotiff(tmpdir, odc_style_xr_dataset):
+    """Ensure the geotiff helper writer works, and supports datasets smaller than 256x256."""
+    filename = tmpdir + '/test.tif'
+
+    assert len(odc_style_xr_dataset.latitude) < 256
+
+    write_geotiff(filename, odc_style_xr_dataset)
+
+    assert filename.exists()
+
+    with rasterio.open(str(filename)) as src:
+        written_data = src.read(1)
+
+        assert (written_data == odc_style_xr_dataset['B10']).all()
+
+
+def test_write_geotiff_time_index_deprecated():
+    """The `time_index` parameter to `write_geotiff()` was a poorly thought out addition and is now deprecated."""
+
+    with pytest.raises(ValueError):
+        write_geotiff("", None, time_index=1)