Repository: fieldsoftheworld/ftw-baselines (language: python)

Issue:
Add basic tests for CLI
We should add some very basic tests that run all commands step by step on a small sample to check we don't break anything.

- [x] Test framework
- [x] Enable CI
- [x] Basic tests for ftw data
- [x] Basic tests for ftw inference
- [x] Basic tests for ftw model


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 9fad49f..79fe60c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,9 @@ build/
 inference_imagery/
 src/ftw.egg-info/
 *.ckpt
-*.tif
+*.aux.xml
+*.gpkg
 .pytest_cache/
 /results.csv
+/austria_example_output_full.gpkg
+/austria_example_output_full.tif


Test patch:
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
index 50837c7..6424f93 100644
--- a/.github/workflows/pytest.yml
+++ b/.github/workflows/pytest.yml
@@ -23,5 +23,15 @@ jobs:
           auto-activate-base: false
       - name: Install Python dependencies
         run: pip install -e .[dev]
-      - name: Execute test suites
-        run: pytest src/tests
+      - name: Download model
+        run: wget https://github.com/fieldsoftheworld/ftw-baselines/releases/download/Pretrained-Models/3_Class_FULL_FTW_Pretrained.ckpt
+      - name: Execute help tests
+        run: pytest src/tests/test_help.py
+      - name: Execute data tests
+        run: pytest src/tests/test_data.py
+      # Seem to be too heavy to run in the CI
+      # - name: Execute model tests
+      #   run: pytest src/tests/test_model.py
+      - name: Execute inference tests
+        run: pytest src/tests/test_inference.py
+      
diff --git a/src/tests/data-files/inference-img.tif b/src/tests/data-files/inference-img.tif
new file mode 100644
index 0000000..0c2e1f4
Binary files /dev/null and b/src/tests/data-files/inference-img.tif differ
diff --git a/src/tests/data-files/mask.tif b/src/tests/data-files/mask.tif
new file mode 100644
index 0000000..94a05b2
Binary files /dev/null and b/src/tests/data-files/mask.tif differ
diff --git a/src/tests/data-files/min_config.yaml b/src/tests/data-files/min_config.yaml
new file mode 100644
index 0000000..080b60f
--- /dev/null
+++ b/src/tests/data-files/min_config.yaml
@@ -0,0 +1,45 @@
+trainer:
+  max_epochs: 1
+  log_every_n_steps: 5
+  accelerator: cpu
+  default_root_dir: logs/FTW-CI
+  devices: 2
+  strategy: ddp_find_unused_parameters_true
+  callbacks:
+    - class_path: lightning.pytorch.callbacks.ModelCheckpoint
+      init_args:
+        monitor: val_loss
+        mode: min
+        save_top_k: 0
+        save_last: true
+        filename: "{epoch}-{val_loss:.2f}"
+model:
+  class_path: ftw.trainers.CustomSemanticSegmentationTask
+  init_args:
+    class_weights: [0.31,0.68]
+    loss: "ce" 
+    model: "unet"
+    backbone: "efficientnet-b3"
+    weights: true
+    patch_weights : false
+    in_channels: 8
+    num_classes: 2
+    num_filters: 64
+    ignore_index: 3
+    lr: 1e-3
+    patience: 100
+data:
+  class_path: ftw.datamodules.FTWDataModule
+  init_args:
+    batch_size: 32
+    num_workers: 4
+    train_countries:
+      - rwanda
+    val_countries:
+      - rwanda
+    test_countries:
+      - rwanda
+  dict_kwargs:
+    root: data/ftw
+    load_boundaries: false
+seed_everything: 7
\ No newline at end of file
diff --git a/src/tests/test_data.py b/src/tests/test_data.py
index cdcb764..dfae876 100644
--- a/src/tests/test_data.py
+++ b/src/tests/test_data.py
@@ -1,20 +1,13 @@
 import os
-import pytest
 from click.testing import CliRunner
 
 from ftw_cli.cli import data_download as download
 from ftw_cli.cli import data_unpack as unpack
 
 
-@pytest.mark.filterwarnings('ignore::FutureWarning')
 def test_data_download():
     runner = CliRunner()
 
-    # Check help
-    result = runner.invoke(download, ["--help"])
-    assert result.exit_code == 0, result.output
-    assert "Usage: download [OPTIONS]" in result.output
-
     # Clean download
     result = runner.invoke(download, ["-f", "--countries=Rwanda", "--no-unpack"])
     assert result.exit_code == 0, result.output
@@ -40,11 +33,6 @@ def test_data_download():
 
 def test_data_unpack():
     runner = CliRunner()
-    
-    # Check help
-    result = runner.invoke(unpack, ["--help"])
-    assert result.exit_code == 0, result.output
-    assert "Usage: unpack [OPTIONS] [INPUT]" in result.output
 
     # Unpack the files
     result = runner.invoke(unpack, [])
diff --git a/src/tests/test_help.py b/src/tests/test_help.py
new file mode 100644
index 0000000..ae9031d
--- /dev/null
+++ b/src/tests/test_help.py
@@ -0,0 +1,51 @@
+from click.testing import CliRunner
+
+from ftw_cli.cli import data_download as download
+from ftw_cli.cli import data_unpack as unpack
+from ftw_cli.cli import inference_download, inference_polygonize, inference_run
+from ftw_cli.cli import model_fit, model_test
+
+# Run all the help functions for a basic CLI check
+# Just ensures imports are correct and that there are no syntax errors in Python files etc
+
+def test_data_download():
+    runner = CliRunner()
+    result = runner.invoke(download, ["--help"])
+    assert result.exit_code == 0, result.output
+    assert "Usage: download [OPTIONS]" in result.output
+
+def test_data_unpack():
+    runner = CliRunner()
+    result = runner.invoke(unpack, ["--help"])
+    assert result.exit_code == 0, result.output
+    assert "Usage: unpack [OPTIONS] [INPUT]" in result.output
+
+def test_inference_download(): # create_input
+    runner = CliRunner()
+    result = runner.invoke(inference_download, ["--help"])
+    assert result.exit_code == 0, result.output
+    assert "Usage: download [OPTIONS]" in result.output
+
+def test_inference_polygonize():
+    runner = CliRunner()
+    result = runner.invoke(inference_polygonize, ["--help"])
+    assert result.exit_code == 0, result.output
+    assert "Usage: polygonize [OPTIONS] INPUT" in result.output
+
+def test_inference_run():
+    runner = CliRunner()
+    result = runner.invoke(inference_run, ["--help"])
+    assert result.exit_code == 0, result.output
+    assert "Usage: run [OPTIONS] INPUT" in result.output
+
+def test_model_fit():
+    runner = CliRunner()
+    result = runner.invoke(model_fit, ["--help"])
+    assert result.exit_code == 0, result.output
+    assert "Usage: fit [OPTIONS] [CLI_ARGS]..." in result.output
+
+def test_model_test():
+    runner = CliRunner()
+    result = runner.invoke(model_test, ["--help"])
+    assert result.exit_code == 0, result.output
+    assert "Usage: test [OPTIONS] [CLI_ARGS]..." in result.output
diff --git a/src/tests/test_inference.py b/src/tests/test_inference.py
index a1831f7..2e0b821 100644
--- a/src/tests/test_inference.py
+++ b/src/tests/test_inference.py
@@ -1,4 +1,6 @@
 
+import os
+
 from click.testing import CliRunner
 
 from ftw_cli.cli import inference_download, inference_polygonize, inference_run
@@ -7,29 +9,60 @@
 def test_inference_download(): # create_input
     runner = CliRunner()
 
-    # Check help
-    result = runner.invoke(inference_download, ["--help"])
+    # Download imagery by ID from MS Planetary Computer
+    inference_image = "inference_imagery/austria_example.tif"
+    result = runner.invoke(inference_download, [
+        "--win_a=S2B_MSIL2A_20210617T100559_R022_T33UUP_20210624T063729",
+        "--win_b=S2B_MSIL2A_20210925T101019_R022_T33UUP_20210926T121923",
+        "-o", inference_image,
+        "-f"
+    ])
     assert result.exit_code == 0, result.output
-    assert "Usage: download [OPTIONS]" in result.output
-
-    # TODO: Add more tests
+    assert "Finished saving window A to file" in result.output
+    assert "Finished saving window B to file" in result.output
+    assert "Finished merging and writing output" in result.output
+    assert os.path.exists(inference_image)
 
-def test_inference_polygonize():
+def test_inference_run():
     runner = CliRunner()
 
-    # Check help
-    result = runner.invoke(inference_polygonize, ["--help"])
-    assert result.exit_code == 0, result.output
-    assert "Usage: polygonize [OPTIONS] INPUT" in result.output
+    # Check required files are present
+    model_path = "3_Class_FULL_FTW_Pretrained.ckpt"
+    assert os.path.exists(model_path)
+    inf_input_path = "./src/tests/data-files/inference-img.tif"
+    assert os.path.exists(inf_input_path)
 
-    # TODO: Add more tests
+    # Run inference
+    inf_output_path = "austria_example_output_full.tif"
+    result = runner.invoke(inference_run, [
+        inf_input_path,
+        "--model", model_path,
+        "--out", inf_output_path,
+        "--gpu", "0",
+        "--overwrite",
+        "--resize_factor", "2",
+        "--overwrite"
+    ])
+    assert result.exit_code == 0, result.output
+    assert "Using custom trainer" in result.output
+    assert "Finished inference and saved output" in result.output
+    assert os.path.exists(inf_output_path)
 
-def test_inference_run():
+def test_inference_polygonize():
     runner = CliRunner()
 
-    # Check help
-    result = runner.invoke(inference_run, ["--help"])
-    assert result.exit_code == 0, result.output
-    assert "Usage: run [OPTIONS] INPUT" in result.output
+    # Check required files are present
+    mask = "./src/tests/data-files/mask.tif"
+    assert os.path.exists(mask)
 
-    # TODO: Add more tests
+    # Polygonize the file
+    out_path = "polygons.gpkg"
+    result = runner.invoke(inference_polygonize, [
+        mask,
+        "-o", out_path,
+        "-f"
+    ])
+    assert result.exit_code == 0, result.output
+    assert "Polygonizing input file:" in result.output
+    assert "Finished polygonizing output" in result.output
+    assert os.path.exists(out_path)
diff --git a/src/tests/test_model.py b/src/tests/test_model.py
index ee75f17..3e5b6ce 100644
--- a/src/tests/test_model.py
+++ b/src/tests/test_model.py
@@ -1,25 +1,41 @@
+import os
 
 from click.testing import CliRunner
 
-from ftw_cli.cli import model_fit, model_test
+from ftw_cli.cli import model_fit, model_test, data_download
 
+CKPT_FILE = "logs/FTW-CI/lightning_logs/version_0/checkpoints/last.ckpt"
+CONFIG_FILE = "src/tests/data-files/min_config.yaml"
 
 def test_model_fit():
     runner = CliRunner()
 
-    # Check help
-    result = runner.invoke(model_fit, ["--help"])
+    # Download required data for the fit command
+    runner.invoke(data_download, ["--countries=Kenya,Rwanda"])
+    assert os.path.exists("data/ftw/kenya")
+    assert os.path.exists("data/ftw/rwanda")
+    assert os.path.exists(CONFIG_FILE)
+    
+    # Run minimal fit
+    result = runner.invoke(model_fit, ["-c", CONFIG_FILE])
     assert result.exit_code == 0, result.output
-    assert "Usage: fit [OPTIONS] [CLI_ARGS]..." in result.output
-
-    # TODO: Add more tests
+    assert "Train countries: ['kenya', 'rwanda']" in result.output
+    assert "Epoch 0: 100%|" in result.output
+    assert "`Trainer.fit` stopped: `max_epochs=1` reached." in result.output
+    assert os.path.exists(CKPT_FILE)
 
 def test_model_test():
     runner = CliRunner()
 
-    # Check help
-    result = runner.invoke(model_test, ["--help"])
+    # Check model for Kenya
+    result = runner.invoke(model_test, [
+        "--gpu", "0",
+        "--model", CKPT_FILE,
+        "--countries", "Kenya", # should be "kenya", but let's test case insensitivity
+        "--out", "results.csv"
+    ])
     assert result.exit_code == 0, result.output
-    assert "Usage: test [OPTIONS] [CLI_ARGS]..." in result.output
-
-    # TODO: Add more tests
+    assert "Running test command" in result.output
+    assert "Created dataloader" in result.output
+    assert "Object level recall: 0.0000" in result.output
+    assert os.path.exists("results.csv")