Repository: opendatacube/datacube-wps (language: python)

Issue:
Improve WIT test coverage
### Expected behaviour
WIT process should be covered by a test

### Actual behaviour
WIT process is untested in CI

### Steps to fix
- Identify GA DEA Collection3 S3 paths to run test on
- Index this data in Sandbox
- Use the collection2 FC test polygon to test WIT process


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 7b247d7..4257338 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,4 +9,5 @@ build
 dist/
 tmp/
 geckodriver.log
-venv/
\ No newline at end of file
+venv/
+artifacts/
\ No newline at end of file
diff --git a/constraints.txt b/constraints.txt
index 845d0bc..c0bb1b5 100644
--- a/constraints.txt
+++ b/constraints.txt
@@ -4,3 +4,4 @@ gunicorn==20.1.0
 pyproj==2.6.1
 flask==1.1.2
 Werkzeug==1.0.1
+click==7.1.2
diff --git a/populate-db.sh b/populate-db.sh
index a89f48e..fbf2ca4 100644
--- a/populate-db.sh
+++ b/populate-db.sh
@@ -1,4 +1,5 @@
 # mangrove cover test datasets
+echo "Index Mangrove Cover"
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/mangrove_cover/v2.0.2/x_13/y_-17/2002/*.yaml' mangrove_cover
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/mangrove_cover/v2.0.2/x_13/y_-16/2002/*.yaml' mangrove_cover
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/mangrove_cover/v2.0.2/x_13/y_-17/2001/*.yaml' mangrove_cover
@@ -12,6 +13,7 @@ s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/mangrove_cover/v
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/mangrove_cover/v2.0.2/x_13/y_-17/2000/*.yaml' mangrove_cover
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/mangrove_cover/v2.0.2/x_13/y_-16/2000/*.yaml' mangrove_cover
 # fractional cover test datasets
+echo "Index Fractional Cover"
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/fractional-cover/fc/v2.2.1/ls8/x_13/y_-41/2019/04/02/*.yaml' ls8_fc_albers
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/fractional-cover/fc/v2.2.1/ls8/x_13/y_-40/2019/04/02/*.yaml' ls8_fc_albers
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/fractional-cover/fc/v2.2.1/ls8/x_13/y_-41/2019/03/17/*.yaml' ls8_fc_albers
@@ -59,6 +61,7 @@ s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/WOfS/WOFLs/v2.1.
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/WOfS/WOFLs/v2.1.5/combined/x_13/y_-40/2019/05/28/*.yaml' wofs_albers
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/WOfS/WOFLs/v2.1.5/combined/x_13/y_-41/2019/05/28/*.yaml' wofs_albers
 # wofs pixel drill test datasets
+echo "Index WOFS Datasets"
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/WOfS/WOFLs/v2.1.5/combined/x_4/y_-32/2000/02/18/*.yaml' wofs_albers
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/WOfS/WOFLs/v2.1.5/combined/x_4/y_-32/2000/01/01/*.yaml' wofs_albers
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/WOfS/WOFLs/v2.1.5/combined/x_4/y_-32/2000/04/22/*.yaml' wofs_albers
@@ -79,3 +82,28 @@ s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/WOfS/WOFLs/v2.1.
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/WOfS/WOFLs/v2.1.5/combined/x_4/y_-32/2000/04/06/*.yaml' wofs_albers
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/WOfS/WOFLs/v2.1.5/combined/x_4/y_-32/2000/12/02/*.yaml' wofs_albers
 s3-to-dc --skip-lineage --no-sign-request 's3://dea-public-data/WOfS/WOFLs/v2.1.5/combined/x_4/y_-32/2000/09/13/*.yaml' wofs_albers
+# WIT test datasets
+echo "Index WIT Datasets"
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/derivative/ga_ls_fc_3/2-5-0/091/085/2019/03/**/*.json' ga_ls_fc_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/derivative/ga_ls_fc_3/2-5-0/091/085/2019/04/**/*.json' ga_ls_fc_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/derivative/ga_ls_fc_3/2-5-0/091/085/2019/05/**/*.json' ga_ls_fc_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/derivative/ga_ls_fc_3/2-5-0/091/085/2019/06/**/*.json' ga_ls_fc_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/derivative/ga_ls_fc_3/2-5-0/091/085/2019/07/**/*.json' ga_ls_fc_3
+
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/derivative/ga_ls_wo_3/1-6-0/091/085/2019/03/**/*.json' ga_ls_wo_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/derivative/ga_ls_wo_3/1-6-0/091/085/2019/04/**/*.json' ga_ls_wo_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/derivative/ga_ls_wo_3/1-6-0/091/085/2019/05/**/*.json' ga_ls_wo_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/derivative/ga_ls_wo_3/1-6-0/091/085/2019/06/**/*.json' ga_ls_wo_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/derivative/ga_ls_wo_3/1-6-0/091/085/2019/07/**/*.json' ga_ls_wo_3
+
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/baseline/ga_ls7e_ard_3/091/085/2019/03/**/*.json' ga_ls7e_ard_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/baseline/ga_ls7e_ard_3/091/085/2019/04/**/*.json' ga_ls7e_ard_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/baseline/ga_ls7e_ard_3/091/085/2019/05/**/*.json' ga_ls7e_ard_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/baseline/ga_ls7e_ard_3/091/085/2019/06/**/*.json' ga_ls7e_ard_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/baseline/ga_ls7e_ard_3/091/085/2019/07/**/*.json' ga_ls7e_ard_3
+
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/baseline/ga_ls8c_ard_3/091/085/2019/03/**/*.json' ga_ls8c_ard_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/baseline/ga_ls8c_ard_3/091/085/2019/04/**/*.json' ga_ls8c_ard_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/baseline/ga_ls8c_ard_3/091/085/2019/05/**/*.json' ga_ls8c_ard_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/baseline/ga_ls8c_ard_3/091/085/2019/06/**/*.json' ga_ls8c_ard_3
+s3-to-dc --skip-lineage --no-sign-request --stac 's3://dea-public-data/baseline/ga_ls8c_ard_3/091/085/2019/07/**/*.json' ga_ls8c_ard_3
\ No newline at end of file
diff --git a/setup-db.sh b/setup-db.sh
index 95922b5..5e0c3c7 100755
--- a/setup-db.sh
+++ b/setup-db.sh
@@ -2,6 +2,6 @@
 docker-compose exec -T index datacube system init --no-default-types --no-init-users
 docker-compose exec -T index datacube metadata add "$METADATA_CATALOG"
 docker-compose exec -T index wget "$PRODUCT_CATALOG" -O product_list.csv
-docker-compose exec -T index bash -c "tail -n+2 product_list.csv | grep 'wofs_albers\|ls._fc_albers\|mangrove_cover' | awk -F , '{print \$2}' | xargs datacube -v product add"
+docker-compose exec -T index bash -c "tail -n+2 product_list.csv | grep 'wofs_albers\|ls._fc_albers\|mangrove_cover\|ga_ls_fc_3\|ga_ls_wo_3\|ga_ls8c_ard_3\|ga_ls7e_ard_3\|ga_ls5t_ard_3' | awk -F , '{print \$2}' | xargs datacube -v product add"
 
 cat populate-db.sh | docker-compose exec -T index bash


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c8733cf..64220c9 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,7 +14,7 @@ env:
   ORG: opendatacube
   IMAGE: wps
   METADATA_CATALOG: https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/a4f39b485b33608a016032d9987251881fec4b6f/workspaces/sandbox-metadata.yaml
-  PRODUCT_CATALOG: https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/a4f39b485b33608a016032d9987251881fec4b6f/workspaces/sandbox-products.csv
+  PRODUCT_CATALOG: https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/fix/collection3_aws/workspaces/sandbox-products.csv
 
 jobs:
   test:
diff --git a/tests/test_api.py b/tests/test_api.py
index e26c7f8..65c5052 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -4,6 +4,7 @@
 from datacube_wps.impl import read_process_catalog
 from datacube_wps.processes.fcdrill import FCDrill
 from datacube_wps.processes.mangrovedrill import MangroveDrill
+from datacube_wps.processes.witprocess import WIT
 from datacube_wps.processes.wofsdrill import WOfSDrill
 
 
@@ -63,3 +64,26 @@ def test_wofs():
     results = wofs.query_handler(time="2000", feature=point)
     assert "data" in results
     assert "chart" in results
+
+
+def test_wit():
+    catalog = read_process_catalog("datacube-wps-config.yaml")
+    wit_proc = [entry for entry in catalog if isinstance(entry, WIT)][0]
+    poly = Geometry(
+        {
+            "type": "Polygon",
+            "coordinates": [
+                [
+                    (147.28271484375003, -35.89238773935897),
+                    (147.03277587890628, -35.663990911348115),
+                    (146.65237426757815, -35.90684930677119),
+                    (147.09182739257815, -36.15894422111004),
+                    (147.28271484375003, -35.89238773935897),
+                ]
+            ],
+        },
+        crs=CRS("EPSG:4326"),
+    )
+    results = wit_proc.query_handler(time=("2019-03-05", "2019-07-10"), feature=poly)
+    assert "data" in results
+    assert "chart" in results