Repository: microsoft/torchgeo

Issue:
Re-think how configs are handled in train.py
Currently configuration to `train.py` is handled with [OmegaConf](https://omegaconf.readthedocs.io/en/2.1_branch/usage.html). This made more sense when the tasks (and accompanying trainer code) were fragmented, as we could easily define per-task configuration. Now that the trainer code that we would like to include in base TorchGeo are being generalized into things like `ClassificationTask` and `SemanticSegmentationTask` _and_ it is clear that more complicated training configurations won't be supported by torchgeo proper, it might make sense to pull out the OmegaConf part, and go with a more simple `argparse` based approach. Bonus: this would also allow us to get rid of a dependency. I'm not sure how exactly the argparse approach would work in all cases but it is worth more thought!

Lightning has a few pieces of docs that can help with this:
- https://pytorch-lightning.readthedocs.io/en/stable/common/trainer.html#trainer-in-python-scripts
- https://pytorch-lightning.readthedocs.io/en/stable/common/hyperparameters.html
- https://pytorch-lightning.readthedocs.io/en/latest/common/lightning_cli.html

Whatever we settle on here should definitely still allow passing arguments via a YAML config file. This allows reproducible benchmark experiment configurations to be saved in source control.


Discussion:
In #226 I did a pretty impressive job of abusing the configs for the sake of testing. Once the release is finished, we'll want to go back through this and clean things up. First order of business is to create a `tests/conf` directory to hold testing-specific config settings so that we don't need to abuse `conf/task_defaults` anymore. Second order of business is to try to reduce some of the duplication between `conf/*.yaml` and `conf/task_defaults/*.yaml`. Third order of business is to try and improve documentation/CLI so it's easier to tell what settings are available. I also want to see if there's some kind of validation thing we could use to ensure that all settings in a config file are actually valid.
In https://github.com/microsoft/torchgeo/pull/286 we changed the number of classes in the task_default config files to make the tests pass. As a side effect, this breaks training runs where the config files don't have the correct number of classes specified. Users shouldn't need to figure out / input the correct number of classes for a dataset in a config file in order to run train.py. 

Resolving patch:
diff --git a/train.py b/train.py
--- a/train.py
+++ b/train.py
@@ -39,28 +39,19 @@
 TASK_TO_MODULES_MAPPING: Dict[
     str, Tuple[Type[pl.LightningModule], Type[pl.LightningDataModule]]
 ] = {
-    "bigearthnet_all": (MultiLabelClassificationTask, BigEarthNetDataModule),
-    "bigearthnet_s1": (MultiLabelClassificationTask, BigEarthNetDataModule),
-    "bigearthnet_s2": (MultiLabelClassificationTask, BigEarthNetDataModule),
+    "bigearthnet": (MultiLabelClassificationTask, BigEarthNetDataModule),
     "byol": (BYOLTask, ChesapeakeCVPRDataModule),
-    "chesapeake_cvpr_5": (SemanticSegmentationTask, ChesapeakeCVPRDataModule),
-    "chesapeake_cvpr_7": (SemanticSegmentationTask, ChesapeakeCVPRDataModule),
-    "chesapeake_cvpr_prior": (SemanticSegmentationTask, ChesapeakeCVPRDataModule),
+    "chesapeake_cvpr": (SemanticSegmentationTask, ChesapeakeCVPRDataModule),
     "cowc_counting": (RegressionTask, COWCCountingDataModule),
     "cyclone": (RegressionTask, CycloneDataModule),
     "eurosat": (ClassificationTask, EuroSATDataModule),
     "etci2021": (SemanticSegmentationTask, ETCI2021DataModule),
     "landcoverai": (SemanticSegmentationTask, LandCoverAIDataModule),
     "naipchesapeake": (SemanticSegmentationTask, NAIPChesapeakeDataModule),
-    "oscd_all": (SemanticSegmentationTask, OSCDDataModule),
-    "oscd_rgb": (SemanticSegmentationTask, OSCDDataModule),
+    "oscd": (SemanticSegmentationTask, OSCDDataModule),
     "resisc45": (ClassificationTask, RESISC45DataModule),
-    "sen12ms_all": (SemanticSegmentationTask, SEN12MSDataModule),
-    "sen12ms_s1": (SemanticSegmentationTask, SEN12MSDataModule),
-    "sen12ms_s2_all": (SemanticSegmentationTask, SEN12MSDataModule),
-    "sen12ms_s2_reduced": (SemanticSegmentationTask, SEN12MSDataModule),
-    "so2sat_supervised": (ClassificationTask, So2SatDataModule),
-    "so2sat_unsupervised": (ClassificationTask, So2SatDataModule),
+    "sen12ms": (SemanticSegmentationTask, SEN12MSDataModule),
+    "so2sat": (ClassificationTask, So2SatDataModule),
     "ucmerced": (ClassificationTask, UCMercedDataModule),
 }
 
@@ -104,7 +95,7 @@ def set_up_omegaconf() -> DictConfig:
     # These OmegaConf structured configs enforce a schema at runtime, see:
     # https://omegaconf.readthedocs.io/en/2.0_branch/structured_config.html#merging-with-other-configs
     task_name = conf.experiment.task
-    task_config_fn = os.path.join("conf", "task_defaults", f"{task_name}.yaml")
+    task_config_fn = os.path.join("conf", f"{task_name}.yaml")
     if task_name == "test":
         task_conf = OmegaConf.create()
     elif os.path.exists(task_config_fn):


Test patch:
diff --git a/conf/task_defaults/bigearthnet_all.yaml b/tests/conf/bigearthnet_all.yaml
similarity index 100%
rename from conf/task_defaults/bigearthnet_all.yaml
rename to tests/conf/bigearthnet_all.yaml
diff --git a/conf/task_defaults/bigearthnet_s1.yaml b/tests/conf/bigearthnet_s1.yaml
similarity index 100%
rename from conf/task_defaults/bigearthnet_s1.yaml
rename to tests/conf/bigearthnet_s1.yaml
diff --git a/conf/task_defaults/bigearthnet_s2.yaml b/tests/conf/bigearthnet_s2.yaml
similarity index 100%
rename from conf/task_defaults/bigearthnet_s2.yaml
rename to tests/conf/bigearthnet_s2.yaml
diff --git a/conf/task_defaults/byol.yaml b/tests/conf/byol.yaml
similarity index 100%
rename from conf/task_defaults/byol.yaml
rename to tests/conf/byol.yaml
diff --git a/conf/task_defaults/chesapeake_cvpr_5.yaml b/tests/conf/chesapeake_cvpr_5.yaml
similarity index 100%
rename from conf/task_defaults/chesapeake_cvpr_5.yaml
rename to tests/conf/chesapeake_cvpr_5.yaml
diff --git a/conf/task_defaults/chesapeake_cvpr_7.yaml b/tests/conf/chesapeake_cvpr_7.yaml
similarity index 100%
rename from conf/task_defaults/chesapeake_cvpr_7.yaml
rename to tests/conf/chesapeake_cvpr_7.yaml
diff --git a/conf/task_defaults/chesapeake_cvpr_prior.yaml b/tests/conf/chesapeake_cvpr_prior.yaml
similarity index 100%
rename from conf/task_defaults/chesapeake_cvpr_prior.yaml
rename to tests/conf/chesapeake_cvpr_prior.yaml
diff --git a/conf/task_defaults/cowc_counting.yaml b/tests/conf/cowc_counting.yaml
similarity index 100%
rename from conf/task_defaults/cowc_counting.yaml
rename to tests/conf/cowc_counting.yaml
diff --git a/conf/task_defaults/cyclone.yaml b/tests/conf/cyclone.yaml
similarity index 100%
rename from conf/task_defaults/cyclone.yaml
rename to tests/conf/cyclone.yaml
diff --git a/conf/task_defaults/etci2021.yaml b/tests/conf/etci2021.yaml
similarity index 100%
rename from conf/task_defaults/etci2021.yaml
rename to tests/conf/etci2021.yaml
diff --git a/conf/task_defaults/eurosat.yaml b/tests/conf/eurosat.yaml
similarity index 100%
rename from conf/task_defaults/eurosat.yaml
rename to tests/conf/eurosat.yaml
diff --git a/conf/task_defaults/landcoverai.yaml b/tests/conf/landcoverai.yaml
similarity index 100%
rename from conf/task_defaults/landcoverai.yaml
rename to tests/conf/landcoverai.yaml
diff --git a/conf/task_defaults/naipchesapeake.yaml b/tests/conf/naipchesapeake.yaml
similarity index 100%
rename from conf/task_defaults/naipchesapeake.yaml
rename to tests/conf/naipchesapeake.yaml
diff --git a/conf/task_defaults/oscd_all.yaml b/tests/conf/oscd_all.yaml
similarity index 100%
rename from conf/task_defaults/oscd_all.yaml
rename to tests/conf/oscd_all.yaml
diff --git a/conf/task_defaults/oscd_rgb.yaml b/tests/conf/oscd_rgb.yaml
similarity index 100%
rename from conf/task_defaults/oscd_rgb.yaml
rename to tests/conf/oscd_rgb.yaml
diff --git a/conf/task_defaults/resisc45.yaml b/tests/conf/resisc45.yaml
similarity index 100%
rename from conf/task_defaults/resisc45.yaml
rename to tests/conf/resisc45.yaml
diff --git a/conf/task_defaults/sen12ms_all.yaml b/tests/conf/sen12ms_all.yaml
similarity index 100%
rename from conf/task_defaults/sen12ms_all.yaml
rename to tests/conf/sen12ms_all.yaml
diff --git a/conf/task_defaults/sen12ms_s1.yaml b/tests/conf/sen12ms_s1.yaml
similarity index 100%
rename from conf/task_defaults/sen12ms_s1.yaml
rename to tests/conf/sen12ms_s1.yaml
diff --git a/conf/task_defaults/sen12ms_s2_all.yaml b/tests/conf/sen12ms_s2_all.yaml
similarity index 100%
rename from conf/task_defaults/sen12ms_s2_all.yaml
rename to tests/conf/sen12ms_s2_all.yaml
diff --git a/conf/task_defaults/sen12ms_s2_reduced.yaml b/tests/conf/sen12ms_s2_reduced.yaml
similarity index 100%
rename from conf/task_defaults/sen12ms_s2_reduced.yaml
rename to tests/conf/sen12ms_s2_reduced.yaml
diff --git a/conf/task_defaults/so2sat_supervised.yaml b/tests/conf/so2sat_supervised.yaml
similarity index 100%
rename from conf/task_defaults/so2sat_supervised.yaml
rename to tests/conf/so2sat_supervised.yaml
diff --git a/conf/task_defaults/so2sat_unsupervised.yaml b/tests/conf/so2sat_unsupervised.yaml
similarity index 100%
rename from conf/task_defaults/so2sat_unsupervised.yaml
rename to tests/conf/so2sat_unsupervised.yaml
diff --git a/conf/task_defaults/ucmerced.yaml b/tests/conf/ucmerced.yaml
similarity index 100%
rename from conf/task_defaults/ucmerced.yaml
rename to tests/conf/ucmerced.yaml
diff --git a/tests/datamodules/test_chesapeake.py b/tests/datamodules/test_chesapeake.py
--- a/tests/datamodules/test_chesapeake.py
+++ b/tests/datamodules/test_chesapeake.py
@@ -14,9 +14,7 @@
 class TestChesapeakeCVPRDataModule:
     @pytest.fixture(scope="class")
     def datamodule(self) -> ChesapeakeCVPRDataModule:
-        conf = OmegaConf.load(
-            os.path.join("conf", "task_defaults", "chesapeake_cvpr_5.yaml")
-        )
+        conf = OmegaConf.load(os.path.join("tests", "conf", "chesapeake_cvpr_5.yaml"))
         kwargs = OmegaConf.to_object(conf.experiment.datamodule)
         kwargs = cast(Dict[str, Any], kwargs)
 
diff --git a/tests/trainers/test_byol.py b/tests/trainers/test_byol.py
--- a/tests/trainers/test_byol.py
+++ b/tests/trainers/test_byol.py
@@ -43,7 +43,7 @@ class TestBYOLTask:
         ],
     )
     def test_trainer(self, name: str, classname: Type[LightningDataModule]) -> None:
-        conf = OmegaConf.load(os.path.join("conf", "task_defaults", name + ".yaml"))
+        conf = OmegaConf.load(os.path.join("tests", "conf", name + ".yaml"))
         conf_dict = OmegaConf.to_object(conf.experiment)
         conf_dict = cast(Dict[Any, Dict[Any, Any]], conf_dict)
 
diff --git a/tests/trainers/test_classification.py b/tests/trainers/test_classification.py
--- a/tests/trainers/test_classification.py
+++ b/tests/trainers/test_classification.py
@@ -47,7 +47,7 @@ def test_trainer(
         if name.startswith("so2sat"):
             pytest.importorskip("h5py")
 
-        conf = OmegaConf.load(os.path.join("conf", "task_defaults", name + ".yaml"))
+        conf = OmegaConf.load(os.path.join("tests", "conf", name + ".yaml"))
         conf_dict = OmegaConf.to_object(conf.experiment)
         conf_dict = cast(Dict[Any, Dict[Any, Any]], conf_dict)
 
@@ -68,7 +68,7 @@ def test_trainer(
         trainer.test(model=model, datamodule=datamodule)
 
     def test_no_logger(self) -> None:
-        conf = OmegaConf.load(os.path.join("conf", "task_defaults", "ucmerced.yaml"))
+        conf = OmegaConf.load(os.path.join("tests", "conf", "ucmerced.yaml"))
         conf_dict = OmegaConf.to_object(conf.experiment)
         conf_dict = cast(Dict[Any, Dict[Any, Any]], conf_dict)
 
@@ -142,7 +142,7 @@ def test_trainer(
         name: str,
         classname: Type[LightningDataModule],
     ) -> None:
-        conf = OmegaConf.load(os.path.join("conf", "task_defaults", name + ".yaml"))
+        conf = OmegaConf.load(os.path.join("tests", "conf", name + ".yaml"))
         conf_dict = OmegaConf.to_object(conf.experiment)
         conf_dict = cast(Dict[Any, Dict[Any, Any]], conf_dict)
 
@@ -163,9 +163,7 @@ def test_trainer(
         trainer.test(model=model, datamodule=datamodule)
 
     def test_no_logger(self) -> None:
-        conf = OmegaConf.load(
-            os.path.join("conf", "task_defaults", "bigearthnet_s1.yaml")
-        )
+        conf = OmegaConf.load(os.path.join("tests", "conf", "bigearthnet_s1.yaml"))
         conf_dict = OmegaConf.to_object(conf.experiment)
         conf_dict = cast(Dict[Any, Dict[Any, Any]], conf_dict)
 
diff --git a/tests/trainers/test_regression.py b/tests/trainers/test_regression.py
--- a/tests/trainers/test_regression.py
+++ b/tests/trainers/test_regression.py
@@ -20,7 +20,7 @@ class TestRegressionTask:
         [("cowc_counting", COWCCountingDataModule), ("cyclone", CycloneDataModule)],
     )
     def test_trainer(self, name: str, classname: Type[LightningDataModule]) -> None:
-        conf = OmegaConf.load(os.path.join("conf", "task_defaults", name + ".yaml"))
+        conf = OmegaConf.load(os.path.join("tests", "conf", name + ".yaml"))
         conf_dict = OmegaConf.to_object(conf.experiment)
         conf_dict = cast(Dict[Any, Dict[Any, Any]], conf_dict)
 
@@ -40,7 +40,7 @@ def test_trainer(self, name: str, classname: Type[LightningDataModule]) -> None:
         trainer.test(model=model, datamodule=datamodule)
 
     def test_no_logger(self) -> None:
-        conf = OmegaConf.load(os.path.join("conf", "task_defaults", "cyclone.yaml"))
+        conf = OmegaConf.load(os.path.join("tests", "conf", "cyclone.yaml"))
         conf_dict = OmegaConf.to_object(conf.experiment)
         conf_dict = cast(Dict[Any, Dict[Any, Any]], conf_dict)
 
diff --git a/tests/trainers/test_segmentation.py b/tests/trainers/test_segmentation.py
--- a/tests/trainers/test_segmentation.py
+++ b/tests/trainers/test_segmentation.py
@@ -50,7 +50,7 @@ def test_trainer(
         name: str,
         classname: Type[LightningDataModule],
     ) -> None:
-        conf = OmegaConf.load(os.path.join("conf", "task_defaults", name + ".yaml"))
+        conf = OmegaConf.load(os.path.join("tests", "conf", name + ".yaml"))
         conf_dict = OmegaConf.to_object(conf.experiment)
         conf_dict = cast(Dict[Any, Dict[Any, Any]], conf_dict)
 
@@ -72,7 +72,7 @@ def test_trainer(
         trainer.test(model=model, datamodule=datamodule)
 
     def test_no_logger(self) -> None:
-        conf = OmegaConf.load(os.path.join("conf", "task_defaults", "landcoverai.yaml"))
+        conf = OmegaConf.load(os.path.join("tests", "conf", "landcoverai.yaml"))
         conf_dict = OmegaConf.to_object(conf.experiment)
         conf_dict = cast(Dict[Any, Dict[Any, Any]], conf_dict)