Repository: opendatacube/datacube-core

Issue:
No validation of config in ingestor
### Expected behaviour

When user supplies mal-formed ingestor configuration I expect errors to be reported to the user, instead exception with `KeyError` is raised.

### Actual behaviour

ingestor does not check that `src_varname` is valid for source product, resulting in errors like following (reported on slack):

```
File "/root/miniconda3/envs/odc/lib/python3.6/site-packages/datacube/scripts/ingest.py", line 153, in ensure_output_type
    output_type = morph_dataset_type(source_type, config, index, storage_format)
  File "/root/miniconda3/envs/odc/lib/python3.6/site-packages/datacube/scripts/ingest.py", line 72, in morph_dataset_type
    for spec in config['measurements']]
  File "/root/miniconda3/envs/odc/lib/python3.6/site-packages/datacube/scripts/ingest.py", line 72, in <listcomp>
    for spec in config['measurements']]
KeyError: '1'
```

### Steps to reproduce the behaviour

Create ingestor config with mis-spelled `src_varname:` attribute for one of the measurements, then run ingestor.

Error is here:

https://github.com/opendatacube/datacube-core/blob/73d98a36d69a1a4aaee8cc89945cf652a6de6260/datacube/scripts/ingest.py#L71-L72

no prior validation happened, yet code assumes that `src_varname` 
(a) exists in the config (maybe that part is checked with schema?)
(b) points to valid measurement in the source product




Discussion:
I will work on this.
We could take advantage of this library for YAML schema validation.

https://pypi.org/project/pykwalify/
http://www.kuwata-lab.com/kwalify/ruby/users-guide.01.html#schema
Thanks @jdh-ama as for schema validation we currently use is `jsonschema`

https://github.com/opendatacube/datacube-core/blob/1b649b59aa66ac6fe655f6dee0c841ff2fba9fa7/datacube/utils/documents.py#L542

Used like this here:

https://github.com/opendatacube/datacube-core/blob/1b649b59aa66ac6fe655f6dee0c841ff2fba9fa7/datacube/model/__init__.py#L394-L395

I'm not familiar with this space and there are some problems with the current approach see #748, but unless proposed library offers significantly better experience I would prefer to use existing approach for validating ingestor config.

Resolving patch:
diff --git a/datacube/model/__init__.py b/datacube/model/__init__.py
--- a/datacube/model/__init__.py
+++ b/datacube/model/__init__.py
@@ -539,6 +539,14 @@ def __hash__(self):
         return hash(self.name)
 
 
+@schema_validated(SCHEMA_PATH / 'ingestor-config-type-schema.yaml')
+class IngestorConfig(object):
+    """
+    Ingestor configuration definition
+    """
+    pass
+
+
 class GridSpec(object):
     """
     Definition for a regular spatial grid
diff --git a/datacube/scripts/ingest.py b/datacube/scripts/ingest.py
--- a/datacube/scripts/ingest.py
+++ b/datacube/scripts/ingest.py
@@ -13,11 +13,12 @@
 import datacube
 from datacube.api.core import Datacube
 from datacube.index.index import Index
-from datacube.model import DatasetType, Range, Measurement
+from datacube.model import DatasetType, Range, Measurement, IngestorConfig
 from datacube.utils import geometry
 from datacube.model.utils import make_dataset, xr_apply, datasets_to_doc
 from datacube.ui import click as ui
 from datacube.utils import read_documents
+from datacube.utils.documents import InvalidDocException
 from datacube.utils.uris import normalise_path
 from datacube.ui.task_app import check_existing_files, load_tasks as load_tasks_, save_tasks as save_tasks_
 from datacube.drivers import storage_writer_by_name
@@ -408,19 +409,34 @@ def ingest_cmd(index,
 
     if config_file:
         config = load_config_from_file(config_file)
+    elif load_tasks:
+        config, tasks = load_tasks_(load_tasks)
+    else:
+        click.echo('Must specify exactly one of --config-file, --load-tasks')
+        sys.exit(-1)
+
+    try:
+        # ignore the added filename key which is not part of the schema
+        filename = config['filename']
+        del config['filename']
+
+        IngestorConfig.validate(config)
+        
+        config['filename'] = filename
+    except InvalidDocException as e:
+        exception, = e.args
+        _LOG.error(exception.message)
+        sys.exit(-1)
+
+    if config_file:
         driver = get_driver_from_config(config)
         source_type, output_type = ensure_output_type(index, config, driver.format,
                                                       allow_product_changes=allow_product_changes)
-
         tasks = create_task_list(index, output_type, year, source_type, config)
     elif load_tasks:
-        config, tasks = load_tasks_(load_tasks)
         driver = get_driver_from_config(config)
         source_type, output_type = ensure_output_type(index, config, driver.format,
                                                       allow_product_changes=allow_product_changes)
-    else:
-        click.echo('Must specify exactly one of --config-file, --load-tasks')
-        sys.exit(-1)
 
     if dry_run:
         check_existing_files(get_filename(config, task['tile_index'], task['tile'].sources) for task in tasks)


Test patch:
diff --git a/integration_tests/data/ingester/invalid_config.yaml b/integration_tests/data/ingester/invalid_config.yaml
new file mode 100644
--- /dev/null
+++ b/integration_tests/data/ingester/invalid_config.yaml
@@ -0,0 +1,178 @@
+source_type: ls5_ledaps_scene
+output_type: ls5_ledaps_albers
+
+description: Landsat 5 LEDAPS 25 metre, 100km tile, Australian Albers Equal Area projection (EPSG:3577)
+
+location: '/media/simonaoliver/datacube/tiles'
+file_path_template: 'LS5_TM_LEDAPS/LS5_TM_LEDAPS_3577_{tile_index[0]}_{tile_index[1]}_{start_time}.nc'
+global_attributes:
+  title: CEOS Data Cube Landsat Surface Reflectance
+  summary: Landsat 5 Thematic Mapper Analysis Ready data prepared by NASA on behalf of CEOS.
+  source: Surface reflectance from LEDAPS
+  institution: CEOS
+  instrument: TM
+  cdm_data_type: Grid
+  keywords: AU/GA,NASA/GSFC/SED/ESD/LANDSAT,REFLECTANCE,ETM+,TM,OLI,EARTH SCIENCE
+  keywords_vocabulary: GCMD
+  platform: LANDSAT-5
+  processing_level: L2
+  product_version: '2.0.0'
+  product_suite: USGS Landsat
+  project: CEOS
+  coverage_content_type: physicalMeasurement
+  references: http://dx.doi.org/10.3334/ORNLDAAC/1146
+  license: https://creativecommons.org/licenses/by/4.0/
+  naming_authority: gov.usgs
+  acknowledgment: Landsat data is provided by the United States Geological Survey (USGS).
+
+storage:
+  driver: NetCDF CF
+
+  crs: EPSG:3577
+  tile_size:
+          x: 100000.0
+          y: 100000.0
+  resolution:
+          x: 25
+          y: -25
+  chunking:
+      x: 200
+      y: 200
+      time: 1
+  dimension_order: ['time', 'y', 'x']
+
+measurements:
+    - name: blue
+      dtype: int16
+      nodata: -9999
+      resampling_method: nearest
+      srx_varname: 'blue'
+      zlib: True
+      attrs:
+          long_name: "Surface Reflectance 0.45-0.52 microns (Blue)"
+          alias: "band_1"
+    - name: green
+      dtype: int16
+      nodata: -9999
+      resampling_method: nearest
+      src_varname: 'green'
+      zlib: True
+      attrs:
+          long_name: "Surface Reflectance 0.52-0.60 microns (Green)"
+          alias: "band_2"
+    - name: red
+      dtype: int16
+      nodata: -9999
+      resampling_method: nearest
+      src_varname: 'red'
+      zlib: True
+      attrs:
+          long_name: "Surface Reflectance 0.63-0.69 microns (Red)"
+          alias: "band_3"
+    - name: nir
+      dtype: int16
+      nodata: -9999
+      resampling_method: nearest
+      src_varname: 'nir'
+      zlib: True
+      attrs:
+          long_name: "Surface Reflectance 0.76-0.90 microns (Near Infrared)"
+          alias: "band_4"
+    - name: swir1
+      dtype: int16
+      nodata: -9999
+      resampling_method: nearest
+      src_varname: 'swir1'
+      zlib: True
+      attrs:
+          long_name: "Surface Reflectance 1.55-1.75 microns (Short-wave Infrared)"
+          alias: "band_5"
+    - name: swir2
+      dtype: int16
+      nodata: -9999
+      resampling_method: nearest
+      src_varname: 'swir2'
+      zlib: True
+      attrs:
+          long_name: "Surface Reflectance 2.08-2.35 microns (Short-wave Infrared)"
+          alias: "band_7"
+    - name: cf_mask
+      dtype: int16
+      nodata: 255
+      resampling_method: nearest
+      src_varname: 'cfmask'
+      zlib: True
+      attrs:
+          long_name: "CFmask"
+    - name: cf_mask_conf
+      dtype: int16
+      nodata: 255
+      resampling_method: nearest
+      src_varname: 'cfmask_conf'
+      zlib: True
+      attrs:
+          long_name: "CFmask Cloud Confidence"
+    - name: adjacent_cloud_qa
+      dtype: int16
+      nodata: 0
+      resampling_method: nearest
+      src_varname: 'sr_adjacent_cloud_qa'
+      zlib: True
+      attrs:
+          long_name: "Adjacent Cloud QA"
+    - name: atmos_opacity
+      dtype: int16
+      nodata: -9999
+      resampling_method: nearest
+      src_varname: 'sr_atmos_opacity'
+      zlib: True
+      attrs:
+          long_name: "Atmospheric Opacity"
+    - name: cloud_qa
+      dtype: int16
+      nodata: 0
+      resampling_method: nearest
+      src_varname: 'sr_cloud_qa'
+      zlib: True
+      attrs:
+          long_name: "Cloud QA"
+    - name: cloud_shadow_qa
+      dtype: int16
+      nodata: 0
+      resampling_method: nearest
+      src_varname: 'sr_cloud_shadow_qa'
+      zlib: True
+      attrs:
+          long_name: "Cloud Shadow QA"
+    - name: ddv_qa
+      dtype: int16
+      nodata: 0
+      resampling_method: nearest
+      src_varname: 'sr_ddv_qa'
+      zlib: True
+      attrs:
+          long_name: "DDV QA"
+    - name: fill_qa
+      dtype: int16
+      nodata: 0
+      resampling_method: nearest
+      src_varname: 'sr_fill_qa'
+      zlib: True
+      attrs:
+          long_name: "Fill QA"
+    - name: land_water_qa
+      dtype: int16
+      nodata: 0
+      resampling_method: nearest
+      src_varname: 'sr_land_water_qa'
+      zlib: True
+      attrs:
+          long_name: "Land Water QA"
+    - name: snow_qa
+      dtype: int16
+      nodata: 0
+      resampling_method: nearest
+      src_varname: 'sr_snow_qa'
+      zlib: True
+      attrs:
+          long_name: "Snow QA"
diff --git a/integration_tests/test_validate_ingestion.py b/integration_tests/test_validate_ingestion.py
new file mode 100644
--- /dev/null
+++ b/integration_tests/test_validate_ingestion.py
@@ -0,0 +1,26 @@
+import pytest
+
+from integration_tests.utils import prepare_test_ingestion_configuration
+from integration_tests.test_end_to_end import PROJECT_ROOT
+
+COMPLIANCE_CHECKER_NORMAL_LIMIT = 2
+
+
+@pytest.mark.timeout(20)
+@pytest.mark.parametrize('datacube_env_name', ('datacube',), indirect=True)
+@pytest.mark.usefixtures('indexed_ls5_scene_products')
+def test_invalid_ingestor_config(clirunner, index, tmpdir, ingest_configs):
+    """
+    Test that the ingestor correctly identifies an invalid ingestor config file.
+
+    Note: We do not need to test valid config files as that is covered by the existing
+          ingestor tests.
+    """
+    config = PROJECT_ROOT / 'integration_tests/data/ingester/invalid_config.yaml'
+    config_path, config = prepare_test_ingestion_configuration(tmpdir, None, config, mode='fast_ingest')
+
+    clirunner([
+        'ingest',
+        '--config-file',
+        str(config_path)
+    ], expect_success=False)