Repository: opendatacube/datacube-core

Issue:
`datacube product list` and `datacube product show` do very similar thing
Default behaviour of `datacube product list` is to **show** all products in yaml format. I would expect to see just a list of product names.

`product list` with `-f tab|csv` works a little bit better for seeing list of all products, but still not ideal as lines are very long, not sure who is the intended audience for that.

One really has to do `datacube product list | grep name:` to get an idea about a **list** of products.

`datacube product show` without product name doesn't work at all, it would make more sense to have this command dump all product YAMLs instead of `datacube product list`




Discussion:
What's even worse output of `product list` is not even compatible with `product add` since it includes `id` field which is specific to a particular DB and is not a property user enters via product yaml.

`product show` doesn't have that problem.
I agree, I use the `datacube product list` command a lot and I very much preferred the formatting in version `1.5.5`
Seems to have been addressed by this commit:

https://github.com/opendatacube/datacube-core/commit/fd5d0124fad8d95369723148a65805a0975c4439

Resolving patch:
diff --git a/datacube/scripts/cli_app.py b/datacube/scripts/cli_app.py
--- a/datacube/scripts/cli_app.py
+++ b/datacube/scripts/cli_app.py
@@ -9,7 +9,7 @@
 import datacube.scripts.dataset
 import datacube.scripts.ingest
 import datacube.scripts.product
-import datacube.scripts.metadata_type
+import datacube.scripts.metadata
 import datacube.scripts.system
 import datacube.scripts.user
 
diff --git a/datacube/scripts/metadata_type.py b/datacube/scripts/metadata.py
similarity index 69%
rename from datacube/scripts/metadata_type.py
rename to datacube/scripts/metadata.py
--- a/datacube/scripts/metadata_type.py
+++ b/datacube/scripts/metadata.py
@@ -1,25 +1,28 @@
 
 import json
 import logging
+import sys
+import yaml
 from pathlib import Path
 
 import click
-from click import echo
+from click import echo, style
 
 from datacube.index.index import Index
 from datacube.ui import click as ui
 from datacube.ui.click import cli
 from datacube.utils import read_documents, InvalidDocException
+from datacube.utils.serialise import SafeDatacubeDumper
 
 _LOG = logging.getLogger('datacube-md-type')
 
 
-@cli.group(name='metadata_type', help='Metadata type commands')
-def metadata_type():
+@cli.group(name='metadata', help='Metadata type commands')
+def this_group():
     pass
 
 
-@metadata_type.command('add')
+@this_group.command('add')
 @click.option('--allow-exclusive-lock/--forbid-exclusive-lock', is_flag=True, default=False,
               help='Allow index to be locked from other users while updating (default: false)')
 @click.argument('files',
@@ -41,7 +44,7 @@ def add_metadata_types(index, allow_exclusive_lock, files):
             continue
 
 
-@metadata_type.command('update')
+@this_group.command('update')
 @click.option(
     '--allow-unsafe/--forbid-unsafe', is_flag=True, default=False,
     help="Allow unsafe updates (default: false)"
@@ -93,23 +96,47 @@ def update_metadata_types(index, allow_unsafe, allow_exclusive_lock, dry_run, fi
                                                                                  len(safe_changes)))
 
 
-@metadata_type.command('show')
-@click.option('-v', '--verbose', is_flag=True)
-@click.argument('metadata_type_name', nargs=1)
+@this_group.command('show')
+@click.option('-f', 'output_format', help='Output format',
+              type=click.Choice(['yaml', 'json']), default='yaml', show_default=True)
+@click.argument('metadata_type_name', nargs=-1)
 @ui.pass_index()
-def show_metadata_type(index, metadata_type_name, verbose):
+def show_metadata_type(index, metadata_type_name, output_format):
     """
     Show information about a metadata type.
     """
-    metadata_type_obj = index.metadata_types.get_by_name(metadata_type_name)
-    if verbose:
-        echo(json.dumps(metadata_type_obj.definition, indent=4))
-    else:
-        print(metadata_type_obj.description)
-        print('Search fields: %s' % ', '.join(sorted(metadata_type_obj.dataset_fields.keys())))
-
 
-@metadata_type.command('list')
+    if len(metadata_type_name) == 0:
+        mm = list(index.metadata_types.get_all())
+    else:
+        mm = []
+        for name in metadata_type_name:
+            m = index.metadata_types.get_by_name(name)
+            if m is None:
+                echo('No such metadata: {}'.format(name))
+                sys.exit(1)
+            else:
+                mm.append(m)
+
+    if len(mm) == 0:
+        echo('No metadata')
+        sys.exit(1)
+
+    if output_format == 'yaml':
+        yaml.dump_all((m.definition for m in mm),
+                      sys.stdout,
+                      Dumper=SafeDatacubeDumper,
+                      default_flow_style=False,
+                      indent=4)
+    elif output_format == 'json':
+        if len(mm) > 1:
+            echo('Can not output more than 1 metadata document in json format')
+            sys.exit(1)
+        m = mm[0]
+        echo(json.dumps(m.definition, indent=4))
+
+
+@this_group.command('list')
 @ui.pass_index()
 def list_metadata_types(index):
     """
@@ -119,7 +146,10 @@ def list_metadata_types(index):
 
     if not metadata_types:
         echo('No metadata types found :(')
-        return
+        sys.exit(1)
 
+    max_w = max(len(m.name) for m in metadata_types)
     for m in metadata_types:
-        echo(m)
+        description_short = m.definition.get('description', '').split('\n')[0]
+        name = '{s:<{n}}'.format(s=m.name, n=max_w)
+        echo(style(name, fg='green') + '  ' + description_short)
diff --git a/datacube/scripts/product.py b/datacube/scripts/product.py
--- a/datacube/scripts/product.py
+++ b/datacube/scripts/product.py
@@ -151,8 +151,15 @@ def _write_tab(products):
 
 
 def _default_lister(products):
+    products = list(products)
+    if len(products) == 0:
+        return
+
+    max_w = max(len(p.name) for p in products)
+
     for prod in products:
-        echo(style(prod.name, fg='green') + '\t' + prod.definition.get('description', ''))
+        name = '{s:<{n}}'.format(s=prod.name, n=max_w)
+        echo(style(name, fg='green') + '  ' + prod.definition.get('description', ''))
 
 
 LIST_OUTPUT_WRITERS = {
@@ -181,15 +188,38 @@ def list_products(dc, output_format):
 @product_cli.command('show')
 @click.option('-f', 'output_format', help='Output format',
               type=click.Choice(['yaml', 'json']), default='yaml', show_default=True)
-@click.argument('product_name', nargs=1)
+@click.argument('product_name', nargs=-1)
 @ui.pass_datacube()
 def show_product(dc, product_name, output_format):
     """
     Show details about a product in the generic index.
     """
-    product = dc.index.products.get_by_name(product_name)
+
+    if len(product_name) == 0:
+        products = list(dc.index.products.get_all())
+    else:
+        products = []
+        for name in product_name:
+            p = dc.index.products.get_by_name(name)
+            if p is None:
+                echo('No such product: {}'.format(name))
+                sys.exit(1)
+            else:
+                products.append(p)
+
+    if len(products) == 0:
+        echo('No products')
+        sys.exit(1)
 
     if output_format == 'yaml':
-        yaml.dump(product.definition, sys.stdout, Dumper=SafeDatacubeDumper, default_flow_style=False, indent=4)
+        yaml.dump_all((p.definition for p in products),
+                      sys.stdout,
+                      Dumper=SafeDatacubeDumper,
+                      default_flow_style=False,
+                      indent=4)
     elif output_format == 'json':
+        if len(products) > 1:
+            echo('Can not output more than 1 product in json format')
+            sys.exit(1)
+        product, *_ = products
         click.echo_via_pager(json.dumps(product.definition, indent=4))


Test patch:
diff --git a/integration_tests/test_config_tool.py b/integration_tests/test_config_tool.py
--- a/integration_tests/test_config_tool.py
+++ b/integration_tests/test_config_tool.py
@@ -45,6 +45,38 @@ def test_add_example_dataset_types(clirunner, initialised_postgres_db, default_m
         assert mappings_count > existing_mappings, "Mapping document was not added: " + str(mapping_path)
         existing_mappings = mappings_count
 
+    result = clirunner(['-v', 'metadata', 'list'])
+    assert result.exit_code == 0
+
+    result = clirunner(['-v', 'metadata', 'show', '-f', 'json', 'eo'],
+                       expect_success=True)
+    assert result.exit_code == 0
+
+    result = clirunner(['-v', 'metadata', 'show'],
+                       expect_success=True)
+    assert result.exit_code == 0
+
+    result = clirunner(['-v', 'product', 'list'])
+    assert result.exit_code == 0
+
+    expect_result = 0 if existing_mappings > 0 else 1
+    result = clirunner(['-v', 'product', 'show'],
+                       expect_success=(expect_result == 0))
+    assert result.exit_code == expect_result
+
+    if existing_mappings > 1:
+        result = clirunner(['-v', 'product', 'show', '-f', 'json'],
+                           expect_success=False)
+        assert result.exit_code == 1
+
+        result = clirunner(['-v', 'product', 'show', '-f', 'json', 'ls8_level1_usgs'],
+                           expect_success=False)
+        assert result.exit_code == 0
+
+        result = clirunner(['-v', 'product', 'show', '-f', 'yaml', 'ls8_level1_usgs'],
+                           expect_success=False)
+        assert result.exit_code == 0
+
 
 def test_error_returned_on_invalid(clirunner, initialised_postgres_db):
     """
diff --git a/integration_tests/test_dataset_add.py b/integration_tests/test_dataset_add.py
--- a/integration_tests/test_dataset_add.py
+++ b/integration_tests/test_dataset_add.py
@@ -195,7 +195,7 @@ def test_dataset_add(dataset_add_configs, index_empty, clirunner):
     assert r.exit_code != 0
     assert 'Found no products' in r.output
 
-    clirunner(['metadata_type', 'add', p.metadata])
+    clirunner(['metadata', 'add', p.metadata])
     clirunner(['product', 'add', p.products])
     clirunner(['dataset', 'add', p.datasets])
     clirunner(['dataset', 'add', p.datasets_bad1])
@@ -277,7 +277,7 @@ def test_dataset_add_ambgious_products(dataset_add_configs, index_empty, clirunn
         'dataset2.yml': yaml.safe_dump(dss[1].doc),
     })
 
-    clirunner(['metadata_type', 'add', p.metadata])
+    clirunner(['metadata', 'add', p.metadata])
     clirunner(['product', 'add', str(prefix/'products.yml')])
 
     pp = list(index.products.get_all())
@@ -310,7 +310,7 @@ def test_dataset_add_with_nans(dataset_add_configs, index_empty, clirunner):
     p = dataset_add_configs
     index = index_empty
 
-    clirunner(['metadata_type', 'add', p.metadata])
+    clirunner(['metadata', 'add', p.metadata])
     clirunner(['product', 'add', p.products])
 
     mk = dataset_maker(0)
@@ -391,7 +391,7 @@ def test_dataset_add_inconsistent_measurements(dataset_add_configs, index_empty,
         'dataset.yml': yaml.safe_dump_all(docs),
     })
 
-    clirunner(['metadata_type', 'add', p.metadata])
+    clirunner(['metadata', 'add', p.metadata])
     r = clirunner(['product', 'add', str(prefix/'products.yml')])
 
     pp = list(index.products.get_all())
@@ -412,7 +412,7 @@ def test_dataset_archive_restore(dataset_add_configs, index_empty, clirunner):
     p = dataset_add_configs
     index = index_empty
 
-    clirunner(['metadata_type', 'add', p.metadata])
+    clirunner(['metadata', 'add', p.metadata])
     clirunner(['product', 'add', p.products])
     clirunner(['dataset', 'add', p.datasets])