Repository: opendatacube/odc-tools (language: python)

Issue:
Replace sat-search with pystac-client
See: https://github.com/stac-utils/pystac-client

stac_api_to_dc.py needs it.


Discussion / hints:

Fix patch:
diff --git a/apps/dc_tools/assets/bootstrap-odc.sh b/apps/dc_tools/assets/bootstrap-odc.sh
index eda764340..63a935879 100755
--- a/apps/dc_tools/assets/bootstrap-odc.sh
+++ b/apps/dc_tools/assets/bootstrap-odc.sh
@@ -13,11 +13,10 @@ set -o nounset
 product_catalog=$1
 metadata_catalog=$2
 
+echo "BOOTSTRAP: Initialising DB."
 datacube system init --no-default-types --no-init-users
 # Created using : datacube metadata list | awk '{print $1}' | xargs datacube metadata show
+echo "BOOTSTRAP: Adding metadata types."
 datacube metadata add "$metadata_catalog"
-python -m wget "$product_catalog" -o product_list.csv
-tail -n+2 product_list.csv | awk -F, '{print $2}' | xargs datacube -v product add
-
-# Clean up
-rm product_list.csv
+echo "BOOTSTRAP: Adding products."
+dc-sync-products "$product_catalog"
diff --git a/apps/dc_tools/odc/apps/dc_tools/add_update_products.py b/apps/dc_tools/odc/apps/dc_tools/add_update_products.py
new file mode 100644
index 000000000..05555e22b
--- /dev/null
+++ b/apps/dc_tools/odc/apps/dc_tools/add_update_products.py
@@ -0,0 +1,86 @@
+"""Add or update a list of ODC products. Intended to be used to
+systematically maintain a CSV of products and synchronise it with
+a database"""
+
+
+import logging
+from csv import DictReader
+from typing import Optional
+
+import click
+import datacube
+import fsspec
+import yaml
+from datacube import Datacube
+from odc.apps.dc_tools.utils import update_if_exists
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s: %(levelname)s: %(message)s', datefmt='%m/%d/%Y %I:%M:%S')
+
+
+def _parse_csv(csv_path: str):
+    with fsspec.open(csv_path, mode="r") as f:
+        reader = DictReader(f)
+        for row in reader:
+            yield row
+
+
+def _get_product(product_path: str):
+    with fsspec.open(product_path, mode="r") as f:
+        return yaml.safe_load(f)
+
+
+def add_update_products(
+    dc: Datacube, csv_path: str, update_if_exists: Optional[bool] = False
+):
+    # Parse csv file
+    names_definitions = _parse_csv(csv_path)
+    new_products = {row["product"]: row["definition"] for row in names_definitions}
+    logging.info(f"Found {len(new_products.keys())} products in the CSV {csv_path}")
+
+    # List existing products
+    products = dc.list_products(with_pandas=False)
+    existing_names = [product["name"] for product in products]
+    logging.info(f"Found {len(existing_names)} products in the Datacube")
+
+    added, updated, failed = 0, 0, 0
+
+    for name in new_products.keys():
+        # Add new products
+        try:
+            if name not in existing_names:
+                product_def = _get_product(new_products[name])
+                dc.index.products.add_document(product_def)
+                added += 1
+                logging.info(f"Added product {name}")
+            # Update existing products, if required
+            elif update_if_exists:
+                product_def = _get_product(new_products[name])
+                dc.index.products.update_document(
+                    product_def, allow_unsafe_updates=True
+                )
+                updated += 1
+                logging.info(f"Updated product {name}")
+        except Exception as e:
+            failed += 1
+            logging.error(f"Failed to add/update product {name} with exception: {e}")
+
+    # Return results
+    return added, updated, failed
+
+
+@click.command("dc-sync-products")
+@click.argument("csv-path", nargs=1)
+@update_if_exists
+def cli(csv_path: str, update_if_exists: bool):
+    # Check we can connect to the Datacube
+    dc = datacube.Datacube(app="add_update_products")
+    logging.info(f"Starting up: connected to Datacube, and update-if-exists is {update_if_exists}")
+
+    # TODO: Add in some QA/QC checks
+    added, updated, failed = add_update_products(dc, csv_path, update_if_exists)
+
+    logging.info(f"Added: {added}, Updated: {updated} and Failed: {failed}")
+
+
+if __name__ == "__main__":
+    cli()
diff --git a/apps/dc_tools/odc/apps/dc_tools/utils.py b/apps/dc_tools/odc/apps/dc_tools/utils.py
index 8f9e15e9d..ce32c02d4 100644
--- a/apps/dc_tools/odc/apps/dc_tools/utils.py
+++ b/apps/dc_tools/odc/apps/dc_tools/utils.py
@@ -71,7 +71,7 @@ class IndexingException(Exception):
     "--update-if-exists",
     is_flag=True,
     default=False,
-    help="If the dataset already exists, update it instead of skipping it.",
+    help="If the dataset or product already exists, update it instead of skipping it.",
 )
 
 allow_unsafe = click.option(
diff --git a/apps/dc_tools/setup.py b/apps/dc_tools/setup.py
index 10a666daa..f80faa2f5 100644
--- a/apps/dc_tools/setup.py
+++ b/apps/dc_tools/setup.py
@@ -1,6 +1,6 @@
 from setuptools import setup
 
-TEST_REQUIREMENTS = ["pytest", "deepdiff"]
+TEST_REQUIREMENTS = ["pytest", "deepdiff", "pytest-depends"]
 
 setup(
     name="odc_apps_dc_tools",
@@ -34,6 +34,7 @@
     },
     entry_points={
         "console_scripts": [
+            "dc-sync-products = odc.apps.dc_tools.add_update_products:cli",
             "dc-index-from-tar = odc.apps.dc_tools.index_from_tar:cli",
             "dc-index-export-md = odc.apps.dc_tools.export_md:cli",
             "s3-to-dc = odc.apps.dc_tools.s3_to_dc:cli",
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 14febb926..f60c87ddd 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -16,30 +16,10 @@ RUN apt-get update \
     graphviz \
     && rm -rf /var/lib/apt/lists/*
 
-RUN groupadd --gid 1000 odc \
-    && useradd --gid 1000 \
-    --uid 1000 \
-    --create-home \
-    --shell /bin/bash -N odc \
-    && adduser odc users \
-    && adduser odc sudo \
-    && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \
-    && mkdir /conf \
-    && install -d /env -g odc -o odc \
-    && install -d /code -g odc -o odc \
-    && true
-
-
-COPY docker/with_bootstrap /usr/local/bin
-ENTRYPOINT ["/usr/local/bin/with_bootstrap"]
-VOLUME ["/code"]
 WORKDIR /code
 
-COPY docker/rr-odc-tools.in docker/constraints.txt docker/requirements.txt /conf
-USER odc
+COPY docker/rr-odc-tools.in docker/constraints.txt docker/requirements.txt /conf/
 RUN --mount=type=bind,target=/src \
     --mount=type=cache,target=/home/odc/.cache/pip,uid=1000,gid=1000 \
     (cd /src && tar c .git libs apps ) | (cd /code && tar x) \
-    && env-build-tool new_no_index /conf/rr-odc-tools.in /conf/constraints.txt /env /src/docker/wheels \
-    && rm -rf /code/*
-USER root
+    && env-build-tool new_no_index /conf/rr-odc-tools.in /conf/constraints.txt /env /src/docker/wheels
diff --git a/docker/constraints.txt b/docker/constraints.txt
index b37c6834a..11d9d8260 100644
--- a/docker/constraints.txt
+++ b/docker/constraints.txt
@@ -155,6 +155,7 @@ responses==0.13.3
 rio-cogeo==2.1.4
 rsa==4.7.2
 s3transfer==0.3.7
+# This is pinned to the wrong version, and it fails. Refactor stac_api_to_dc to remove dep.
 sat-search==0.1.0
 scikit-image==0.18.1
 scipy==1.6.3
diff --git a/docker/with_bootstrap b/docker/with_bootstrap
deleted file mode 100755
index ede4ebddc..000000000
--- a/docker/with_bootstrap
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-
-
-# If current work directory is not owned by root
-#   1. Update odc user/group to have same UID,GID as work directory
-#   2. Relaunch this script as odc user
-[[ $UID -ne 0 ]] || {
-    target_uid=$(stat -c '%u' .)
-    target_gid=$(stat -c '%g' .)
-
-    [[ $target_uid -eq 0 ]] || {
-        groupmod --gid "${target_gid}" odc 2> /dev/null > /dev/null
-        usermod --gid "${target_gid}" --uid "${target_uid}" odc > /dev/null 2> /dev/null
-        chown -R odc:odc /home/odc/
-        exec sudo -u odc -E -H bash "$0" "$@"
-    }
-}
-
-[[ $UID -ne 0 ]] || echo "WARNING: Running as root"
-
-# point pip cache to /wk volume
-export PIP_CACHE_DIR=/wk/.cache/pip
-
-case "${1:-help}" in
-    help)
-        echo "TODO: help"
-        ;;
-    python|python3|pip|pip3|pip-compile|pytest)
-        source /env/bin/activate
-        exec "$@"
-        ;;
-    *)
-        exec "$@"
-        ;;
-esac


Test patch:
diff --git a/.github/workflows/build-test-docker.yml b/.github/workflows/build-test-docker.yml
index a9cb06dec..bf5c1ccac 100644
--- a/.github/workflows/build-test-docker.yml
+++ b/.github/workflows/build-test-docker.yml
@@ -60,11 +60,13 @@ jobs:
         docker pull opendatacube/odc-test-runner:latest || true
         make -C docker dkr-no-deps
 
-    - name: Run tests
-      timeout-minutes: 10
+    - name: Run tests and integration tests
+      timeout-minutes: 20
+      shell: bash
       run: |
         cd tests
         docker-compose up -d
+        docker-compose exec -T tools-tester datacube system init
         docker-compose exec -T tools-tester pytest --timeout=30
 
     - name: DockerHub Login
@@ -77,4 +79,3 @@ jobs:
       if: steps.cfg.outputs.push_image == 'yes'
       run: |
         docker push opendatacube/odc-test-runner:latest
-
diff --git a/.github/workflows/test-dc-tools.yml b/.github/workflows/test-dc-tools.yml
index 0da34cf7a..3d60c7ee9 100644
--- a/.github/workflows/test-dc-tools.yml
+++ b/.github/workflows/test-dc-tools.yml
@@ -21,12 +21,17 @@ jobs:
     - name: Run Dockerized Tests
       timeout-minutes: 20
       shell: bash
-      # TODO: Fix superlong line below
       run: |
         cd tests
         docker-compose up -d
-        docker-compose exec -T tools-tester bash -c "pip install wget && /code/apps/dc_tools/assets/bootstrap-odc.sh \$PRODUCT_CATALOG \$METADATA_CATALOG" || true
-        docker-compose exec -T tools-tester datacube product add https://raw.githubusercontent.com/digitalearthafrica/config/78dfc5a1895b48225803cb0891b99eb01feb9f2e/products/esa_s2_l2a.odc-product.yaml
+        # Manually install over the top
+        docker-compose exec -T tools-tester pip install -r /code/docker/rr-odc-tools.in
+        # Prove that worked ^
+        dc-sync-products --help
+        # Now set up some metadata and products
+        docker-compose exec -T tools-tester bash -c \
+          "/code/apps/dc_tools/assets/bootstrap-odc.sh \$PRODUCT_CATALOG \$METADATA_CATALOG"
+        # And run the real tests
         docker-compose exec -T tools-tester pytest --cov=/code/ --cov-report xml:coverage.xml --timeout=30
 
     - name: Upload Coverage
diff --git a/apps/dc_tools/tests/data/example_product_list.csv b/apps/dc_tools/tests/data/example_product_list.csv
new file mode 100644
index 000000000..31007d830
--- /dev/null
+++ b/apps/dc_tools/tests/data/example_product_list.csv
@@ -0,0 +1,3 @@
+product,definition
+s2_l2a,https://raw.githubusercontent.com/digitalearthafrica/config/master/products/esa_s2_l2a.odc-product.yaml
+cemp_insar_alos_displacement,https://raw.githubusercontent.com/GeoscienceAustralia/digitalearthau/885e3d3a00bcc9c9fe3ed43e82201d7121898aed/digitalearthau/config/products/cemp_insar_alos_displacement.yaml
diff --git a/apps/dc_tools/tests/test_add_update_products.py b/apps/dc_tools/tests/test_add_update_products.py
new file mode 100644
index 000000000..c1139deb8
--- /dev/null
+++ b/apps/dc_tools/tests/test_add_update_products.py
@@ -0,0 +1,62 @@
+from pathlib import Path
+
+import pytest
+from click.testing import CliRunner
+from odc.apps.dc_tools.add_update_products import _get_product, _parse_csv, cli
+
+TEST_DATA_FOLDER: Path = Path(__file__).parent.joinpath("data")
+LOCAL_EXAMPLE: str = "example_product_list.csv"
+REMOTE_EXAMPLE: str = (
+    "https://raw.githubusercontent.com/GeoscienceAustralia/"
+    "dea-config/master/workspaces/dev-products.csv"
+)
+PRODUCT_EXAMPLE: str = (
+    "https://raw.githubusercontent.com/digitalearthafrica/"
+    "config/master/products/esa_s2_l2a.odc-product.yaml"
+)
+
+
+def test_parse_csv(local_csv, remote_csv):
+    local_contents = [x for x in _parse_csv(local_csv)]
+
+    assert len(local_contents) == 2
+    assert local_contents[0]["product"] == "s2_l2a"
+    assert local_contents[0]["definition"] == PRODUCT_EXAMPLE
+
+    remote_contents = [x for x in _parse_csv(remote_csv)]
+    assert len(remote_contents) >= 65
+
+
+def test_load_product_def(remote_product):
+    product = _get_product(remote_product)
+
+    assert product["name"] == "s2_l2a"
+
+
+@pytest.mark.depends(name='add_products')
+def test_add_products(local_csv):
+    runner = CliRunner()
+    # This will fail if requester pays is enabled
+    result = runner.invoke(
+        cli,
+        [
+            local_csv,
+            "--update-if-exists",
+        ],
+    )
+    assert result.exit_code == 0
+
+
+@pytest.fixture
+def remote_product():
+    return PRODUCT_EXAMPLE
+
+
+@pytest.fixture
+def local_csv():
+    return str(TEST_DATA_FOLDER / LOCAL_EXAMPLE)
+
+
+@pytest.fixture
+def remote_csv():
+    return REMOTE_EXAMPLE
diff --git a/apps/dc_tools/tests/test_s3_to_dc.py b/apps/dc_tools/tests/test_s3_to_dc.py
index 756f357a3..a5f947073 100644
--- a/apps/dc_tools/tests/test_s3_to_dc.py
+++ b/apps/dc_tools/tests/test_s3_to_dc.py
@@ -1,28 +1,38 @@
 # Tests using the Click framework the s3-to-dc CLI tool
 
+import pytest
 from click.testing import CliRunner
-
 from odc.apps.dc_tools.s3_to_dc import cli
 
+
+@pytest.mark.depends(on=['add_products'])
 def test_s3_to_dc_yaml():
-  runner = CliRunner()
-  # This will fail if requester pays is enabled
-  result = runner.invoke(cli, [
-      "--no-sign-request",
-      "s3://dea-public-data/cemp_insar/insar/displacement/alos/2010/**/*.yaml",
-      "cemp_insar_alos_displacement"
-    ])
-  assert result.exit_code == 0
-  assert result.output == 'Added 25 Datasets, Failed 0 Datasets\n'
+    runner = CliRunner()
+    # This will fail if requester pays is enabled
+    result = runner.invoke(
+        cli,
+        [
+            "--no-sign-request",
+            "s3://dea-public-data/cemp_insar/insar/displacement/alos/2010/**/*.yaml",
+            "cemp_insar_alos_displacement",
+        ],
+    )
+    assert result.exit_code == 0
+    assert result.output == "Added 25 Datasets, Failed 0 Datasets\n"
+
 
+@pytest.mark.depends(on=['add_products'])
 def test_s3_to_dc_stac():
-  runner = CliRunner()
-  # This will fail if requester pays is enabled
-  result = runner.invoke(cli, [
-      "--no-sign-request",
-      "--stac",
-      "s3://sentinel-cogs/sentinel-s2-l2a-cogs/2020/S2A_32NNF_20200127_0_L2A/*.json",
-      "s2_l2a"
-    ])
-  assert result.exit_code == 0
-  assert result.output == 'Added 1 Datasets, Failed 0 Datasets\n'
\ No newline at end of file
+    runner = CliRunner()
+    # This will fail if requester pays is enabled
+    result = runner.invoke(
+        cli,
+        [
+            "--no-sign-request",
+            "--stac",
+            "s3://sentinel-cogs/sentinel-s2-l2a-cogs/2020/S2A_32NNF_20200127_0_L2A/*.json",
+            "s2_l2a",
+        ],
+    )
+    assert result.exit_code == 0
+    assert result.output == "Added 1 Datasets, Failed 0 Datasets\n"
diff --git a/apps/dc_tools/tests/test_stac_api_to_dc.py b/apps/dc_tools/tests/test_stac_api_to_dc.py
index a874b3cbb..5faa9eecc 100644
--- a/apps/dc_tools/tests/test_stac_api_to_dc.py
+++ b/apps/dc_tools/tests/test_stac_api_to_dc.py
@@ -1,1 +1,24 @@
-# Tests using the Click framework the stac_api-to-dc CLI tool
\ No newline at end of file
+# Tests using the Click framework the stac_api-to-dc CLI tool
+import pytest
+from click.testing import CliRunner
+from odc.apps.dc_tools.stac_api_to_dc import cli
+
+
+@pytest.mark.skip(
+    reason="Don't run this, because it needs version 0.3.0 of sat-search, which breaks the build."
+)
+@pytest.mark.depends(on=['add_products'])
+def test_s3_to_dc_stac():
+    runner = CliRunner()
+    # This will fail if requester pays is enabled
+    result = runner.invoke(
+        cli,
+        [
+            "--bbox=5,15,10,20",
+            "--limit=10",
+            "--collections=sentinel-s2-l2a-cogs",
+            "--datetime=2020-08-01/2020-08-31",
+        ],
+    )
+    assert result.exit_code == 0
+    assert result.output == "Added 10 Datasets, failed 0 Datasets\n"