Repository: opendatacube/datacube-core

Issue:
Propose new location management feature for `datacube update`
Currently if you make a copy of ingested file and then call `datacube update <copied-file>` new location for all datasets present in this file (and also in the database) will be added to the index. Previous location will still be present, i.e. `dataset.uris` will become `[new_uri, old_uri]`.

I propose we add an option to archive or forget completely old location, something like

```
--previous-location [keep|archive|forget]
                       What to do with previously recorded dataset location
                       'keep'    -- no change, keep as alternative location [default]
                       'archive' -- mark as archived
                       'forget'  -- remove from the index
``` 

This can then be used to update locations of stacked datasets on NCI with `forget` option.

`archive` and `forget` should probably error out if there is more than 1 active location.



Discussion:

Resolving patch:
diff --git a/datacube/scripts/dataset.py b/datacube/scripts/dataset.py
--- a/datacube/scripts/dataset.py
+++ b/datacube/scripts/dataset.py
@@ -146,20 +146,22 @@ def load_datasets_for_update(datasets, index):
     dataset in the index. Datasets not in the database will be logged.
 
     Doesn't load lineage information
+
+    Generates tuples in the form (new_dataset, existing_dataset)
     """
     def mk_dataset(metadata_doc, uri):
         uuid = metadata_doc.get('id', None)
 
         if uuid is None:
-            return None, "Metadata document it missing id field"
+            return None, None, "Metadata document it missing id field"
 
         existing = index.datasets.get(uuid)
         if existing is None:
-            return None, "No such dataset the database: {}".format(uuid)
+            return None, None, "No such dataset the database: {}".format(uuid)
 
         return Dataset(existing.type,
                        without_lineage_sources(metadata_doc, existing.type, inplace=True),
-                       uris=[uri]), None
+                       uris=[uri]), existing, None
 
     for dataset_path in datasets:
         metadata_path = get_metadata_path(Path(dataset_path))
@@ -169,7 +171,7 @@ def mk_dataset(metadata_doc, uri):
 
         try:
             for uri, metadata_doc in read_documents(metadata_path, uri=True):
-                dataset, error_msg = mk_dataset(metadata_doc, uri)
+                dataset, existing, error_msg = mk_dataset(metadata_doc, uri)
 
                 if dataset is None:
                     _LOG.error("Failure while processing: %s\n" +
@@ -177,7 +179,7 @@ def mk_dataset(metadata_doc, uri):
                 else:
                     is_consistent, reason = check_dataset_consistent(dataset)
                     if is_consistent:
-                        yield dataset
+                        yield dataset, existing
                     else:
                         _LOG.error("Dataset %s inconsistency: %s", dataset.id, reason)
 
@@ -236,45 +238,96 @@ def index_dataset_paths(sources_policy, dry_run, index, rules, dataset_paths):
                 _LOG.error('Failed to add dataset %s: %s', dataset.local_uri, e)
 
 
-def parse_update_rules(allow_any):
-    updates = {}
-    for key_str in allow_any:
-        updates[tuple(key_str.split('.'))] = changes.allow_any
-    return updates
+def parse_update_rules(keys_that_can_change):
+    updates_allowed = {}
+    for key_str in keys_that_can_change:
+        updates_allowed[tuple(key_str.split('.'))] = changes.allow_any
+    return updates_allowed
 
 
 @dataset_cmd.command('update', help="Update datasets in the Data Cube")
-@click.option('--allow-any', help="Allow any changes to the specified key (a.b.c)", multiple=True)
+@click.option('--allow-any', 'keys_that_can_change',
+              help="Allow any changes to the specified key (a.b.c)",
+              multiple=True)
 @click.option('--dry-run', help='Check if everything is ok', is_flag=True, default=False)
+@click.option('--location-policy',
+              type=click.Choice(['keep', 'archive', 'forget']),
+              default='keep',
+              help='''What to do with previously recorded dataset location
+'keep' - keep as alternative location [default]
+'archive' - mark as archived
+'forget' - remove from the index
+''')
 @click.argument('datasets',
                 type=click.Path(exists=True, readable=True, writable=False), nargs=-1)
 @ui.pass_index()
-def update_cmd(index, allow_any, dry_run, datasets):
-    updates = parse_update_rules(allow_any)
+def update_cmd(index, keys_that_can_change, dry_run, location_policy, datasets):
+
+    def loc_action(action, new_ds, existing_ds, action_name):
+        if len(existing_ds.uris) == 0:
+            return None
+
+        if len(existing_ds.uris) > 1:
+            _LOG.warning("Refusing to %s old location, there are several", action_name)
+            return None
+
+        new_uri, = new_ds.uris
+        old_uri, = existing_ds.uris
+
+        if new_uri == old_uri:
+            return None
+
+        if dry_run:
+            echo('Will {} old location {}, and add new one {}'.format(action_name, old_uri, new_uri))
+            return True
+
+        return action(existing_ds.id, old_uri)
+
+    def loc_archive(new_ds, existing_ds):
+        return loc_action(index.datasets.archive_location, new_ds, existing_ds, 'archive')
+
+    def loc_forget(new_ds, existing_ds):
+        return loc_action(index.datasets.remove_location, new_ds, existing_ds, 'forget')
+
+    def loc_keep(new_ds, existing_ds):
+        return None
+
+    update_loc = dict(archive=loc_archive,
+                      forget=loc_forget,
+                      keep=loc_keep)[location_policy]
+
+    updates_allowed = parse_update_rules(keys_that_can_change)
 
     success, fail = 0, 0
-    for dataset in load_datasets_for_update(datasets, index):
+    for dataset, existing_ds in load_datasets_for_update(datasets, index):
         _LOG.info('Matched %s', dataset)
 
+        if location_policy != 'keep':
+            if len(existing_ds.uris) > 1:
+                # TODO:
+                pass
+
         if not dry_run:
             try:
-                index.datasets.update(dataset, updates_allowed=updates)
+                index.datasets.update(dataset, updates_allowed=updates_allowed)
+                update_loc(dataset, existing_ds)
                 success += 1
                 echo('Updated %s' % dataset.id)
             except ValueError as e:
                 fail += 1
                 echo('Failed to update %s: %s' % (dataset.id, e))
         else:
-            if update_dry_run(index, updates, dataset):
+            if update_dry_run(index, updates_allowed, dataset):
+                update_loc(dataset, existing_ds)
                 success += 1
             else:
                 fail += 1
     echo('%d successful, %d failed' % (success, fail))
 
 
-def update_dry_run(index, updates, dataset):
+def update_dry_run(index, updates_allowed, dataset):
     try:
-        can_update, safe_changes, unsafe_changes = index.datasets.can_update(dataset, updates_allowed=updates)
+        can_update, safe_changes, unsafe_changes = index.datasets.can_update(dataset, updates_allowed=updates_allowed)
     except ValueError as e:
         echo('Cannot update %s: %s' % (dataset.id, e))
         return False
diff --git a/datacube/storage/storage.py b/datacube/storage/storage.py
--- a/datacube/storage/storage.py
+++ b/datacube/storage/storage.py
@@ -430,7 +430,7 @@ def get_bandnumber(self, src=None):
         if src.count == 1:  # Single-slice netcdf file
             return 1
 
-        _LOG.warning("Encountered stacked netcdf file without recorded index\n - %s", src.name)
+        _LOG.debug("Encountered stacked netcdf file without recorded index\n - %s", src.name)
 
         # Below is backwards compatibility code
 


Test patch:
diff --git a/integration_tests/test_end_to_end.py b/integration_tests/test_end_to_end.py
--- a/integration_tests/test_end_to_end.py
+++ b/integration_tests/test_end_to_end.py
@@ -87,12 +87,21 @@ def test_end_to_end(clirunner, index, testdata_dir, ingest_configs):
                str(lbg_nbar), str(lbg_pq)])
 
     # Test no-op update
-    clirunner(['-v', 'dataset', 'update', '--dry-run',
-               str(lbg_nbar), str(lbg_pq)])
-
-    # Test no changes needed update
-    clirunner(['-v', 'dataset', 'update',
-               str(lbg_nbar), str(lbg_pq)])
+    for policy in ['archive', 'forget', 'keep']:
+        clirunner(['-v', 'dataset', 'update',
+                   '--dry-run',
+                   '--location-policy', policy,
+                   str(lbg_nbar), str(lbg_pq)])
+
+        # Test no changes needed update
+        clirunner(['-v', 'dataset', 'update',
+                   '--location-policy', policy,
+                   str(lbg_nbar), str(lbg_pq)])
+
+    # TODO: test location update
+    # 1. Make a copy of a file
+    # 2. Call dataset update with archive/forget
+    # 3. Check location
 
     # Ingest NBAR
     clirunner(['-v', 'ingest', '-c', str(ls5_nbar_albers_ingest_config)])