Repository: opendatacube/datacube-core

Issue:
Driver resolution code can not handle dataset format being None
When attempting to load dataset without format specified `None` is used as a string

Example reported on Slack:

```
2020-07-23 17:05:21,576 48721 datacube-ingest ERROR Failed to create storage unit file (Exception: 'NoneType' object has no attribute 'lower') 
Traceback (most recent call last):
  File "/home/sopon/projects/cornucopia/odc-core/datacube/scripts/ingest.py", line 340, in process_tasks
    executor.result(future)
  File "/home/sopon/projects/cornucopia/odc-core/datacube/executor.py", line 63, in result
    return func(*args, **kwargs)
  File "/home/sopon/projects/cornucopia/odc-core/datacube/executor.py", line 33, in reraise
    raise t.with_traceback(e, traceback)
  File "/home/sopon/projects/cornucopia/odc-core/datacube/executor.py", line 36, in get_ready
    result = SerialExecutor.result(futures[0])
  File "/home/sopon/projects/cornucopia/odc-core/datacube/executor.py", line 63, in result
    return func(*args, **kwargs)
  File "/home/sopon/projects/cornucopia/odc-core/datacube/scripts/ingest.py", line 257, in ingest_work
    fuse_func=fuse_func)
  File "/home/sopon/projects/cornucopia/odc-core/datacube/api/core.py", line 589, in load_data
    progress_cbk=progress_cbk)
  File "/home/sopon/projects/cornucopia/odc-core/datacube/api/core.py", line 526, in _xr_load
    progress_cbk=_cbk)
  File "/home/sopon/projects/cornucopia/odc-core/datacube/api/core.py", line 703, in _fuse_measurement
    src = new_datasource(BandInfo(ds, measurement.name))
  File "/home/sopon/projects/cornucopia/odc-core/datacube/drivers/readers.py", line 101, in new_datasource
    source_type = choose_datasource(band)
  File "/home/sopon/projects/cornucopia/odc-core/datacube/drivers/readers.py", line 80, in choose_datasource
    return rdr_cache()(band.uri_scheme, band.format, fallback=RasterDatasetDataSource)
  File "/home/sopon/projects/cornucopia/odc-core/datacube/drivers/readers.py", line 37, in __call__
    driver = self._find_driver(uri_scheme, fmt)
  File "/home/sopon/projects/cornucopia/odc-core/datacube/drivers/readers.py", line 25, in _find_driver
    key = (uri_scheme.lower(), fmt.lower())
AttributeError: 'NoneType' object has no attribute 'lower'
2020-07-23 17:05:21,576 48721 datacube-ingest INFO Storage unit file creation status (Created_Count: 14, Failed_Count: 15)
```

I think proper solution to that is to fallback to default driver implementation. Also at index time warning should be printed if dataset format is not specified, see #955.


Discussion:
Sample dataset as supplied in error report on Slack:

```yaml
---
# Dataset
$schema: https://schemas.opendatacube.org/dataset
id: 815b5ad0-a965-4b24-bc0f-ce508b7b5c5b

label: drone_data_test

product:
  name: drone_data

crs: epsg:32611

geometry:
  type: Polygon
  coordinates: [[[490615.8747233282, 5209843.73], [490506.5596645922, 5209841.102774615],
      [490301.9893593582, 5209835.450946526], [490300.5097876491, 5209835.384700488],
      [490300.29, 5209835.284409483], [490300.29, 5209232.5705832625], [490305.9197764892,
        5209032.714124906], [490311.5718348812, 5208993.148827706], [490313.73707562976,
        5208988.219795475], [490314.0177741324, 5208988.118742354], [490317.2441298913,
        5208987.456011741], [490450.1071950689, 5208978.978241526], [491102.52762041555,
        5208953.567075848], [491184.83496150543, 5208953.567075848], [491187.65174461267,
        5208956.391694053], [491190.4556277827, 5209004.080111855], [491190.4556277827,
        5209468.4927358925], [491176.3481077613, 5209843.73], [490615.8747233282,
        5209843.73]]]
grids:
  default:
    shape: [40320, 40320]
    transform: [0.022077520530325516, 0.0, 490300.29, 0.0, -0.022077453476015127,
      5209843.73, 0.0, 0.0, 1.0]

properties:
  datetime: 2020-04-17 02:47:11.803788Z
  odc:processing_datetime: 2020-07-23 05:52:45.163604Z
  odc:product_family: drone_data

measurements:
  blue:
    path: drone_data_2020-04-17_blue.tif
  green:
    path: drone_data_2020-04-17_green.tif
  red:
    path: drone_data_2020-04-17_red.tif

accessories:
  checksum:sha1:
    path: drone_data_2020-04-17.sha1
  metadata:processor:
    path: drone_data_2020-04-17.proc-info.yaml

lineage: {}
...
```

Resolving patch:
diff --git a/datacube/drivers/readers.py b/datacube/drivers/readers.py
--- a/datacube/drivers/readers.py
+++ b/datacube/drivers/readers.py
@@ -62,7 +62,7 @@ def reader_drivers() -> List[str]:
     return rdr_cache().drivers()
 
 
-def choose_datasource(band: 'BandInfo') -> DatasourceFactory:
+def choose_datasource(band: BandInfo) -> DatasourceFactory:
     """Returns appropriate `DataSource` class (or a constructor method) for loading
     given `dataset`.
 
diff --git a/datacube/storage/_base.py b/datacube/storage/_base.py
--- a/datacube/storage/_base.py
+++ b/datacube/storage/_base.py
@@ -104,9 +104,9 @@ def __init__(self,
         self.units = mp.units
         self.crs = ds.crs
         self.transform = ds.transform
-        self.format = ds.format
+        self.format = ds.format or ''
         self.driver_data = _extract_driver_data(ds)
 
     @property
     def uri_scheme(self) -> str:
-        return urlparse(self.uri).scheme
+        return urlparse(self.uri).scheme or ''


Test patch:
diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py
--- a/tests/storage/test_base.py
+++ b/tests/storage/test_base.py
@@ -64,3 +64,13 @@ def test_band_info():
     ds.uris = None
     with pytest.raises(ValueError):
         BandInfo(ds, 'a')
+
+    ds_none_fmt = mk_sample_dataset(bands,
+                                    uri='file:///tmp/datataset.yml',
+                                    format=None)
+    assert ds_none_fmt.format is None
+    assert BandInfo(ds_none_fmt, 'a').format == ''
+
+    ds = mk_sample_dataset(bands, uri='/not/a/uri')
+    band = BandInfo(ds, 'a')
+    assert(band.uri_scheme is '')
diff --git a/tests/test_driver.py b/tests/test_driver.py
--- a/tests/test_driver.py
+++ b/tests/test_driver.py
@@ -23,6 +23,12 @@ def test_new_datasource_fallback():
     assert rdr is not None
     assert isinstance(rdr, RasterDatasetDataSource)
 
+    # check that None format works
+    band = BandInfo(mk_sample_dataset(bands, 'file:///file', format=None), 'green')
+    rdr = new_datasource(band)
+    assert rdr is not None
+    assert isinstance(rdr, RasterDatasetDataSource)
+
 
 def test_reader_drivers():
     available_drivers = reader_drivers()