Repository: mlcommons/GaNDLF

Issue:
Allow a specific testing data CSV to passed in main CLI
**Is your feature request related to a problem? Please describe.**
Currently, only training and validation CSVs can be passed in CLI, but not testing data. This is useful when the testing data is defined for a particular problem.

**Describe the solution you'd like**
Allow a specific testing data CSV to passed in main CLI

**Describe alternatives you've considered**
Do it after training.

**Additional context**
N.A.


Discussion:

Resolving patch:
diff --git a/GANDLF/cli/main_run.py b/GANDLF/cli/main_run.py
--- a/GANDLF/cli/main_run.py
+++ b/GANDLF/cli/main_run.py
@@ -49,23 +49,32 @@ def main_run(data_csv, config_file, output_dir, train_mode, device, resume, rese
     if "-1" in device:
         device = "cpu"
 
-    if train_mode:  # train mode
+    if train_mode:
         Path(parameters["output_dir"]).mkdir(parents=True, exist_ok=True)
 
     # parse training CSV
     if "," in file_data_full:
         # training and validation pre-split
         data_full = None
-        both_csvs = file_data_full.split(",")
-        data_train, headers_train = parseTrainingCSV(both_csvs[0], train=train_mode)
+        all_csvs = file_data_full.split(",")
+        data_train, headers_train = parseTrainingCSV(all_csvs[0], train=train_mode)
         data_validation, headers_validation = parseTrainingCSV(
-            both_csvs[1], train=train_mode
+            all_csvs[1], train=train_mode
         )
+        assert (
+            headers_train == headers_validation
+        ), "The training and validation CSVs do not have the same header information."
 
-        if headers_train != headers_validation:
-            sys.exit(
-                "The training and validation CSVs do not have the same header information."
+        # testing data is present
+        data_testing = None
+        headers_testing = headers_train
+        if len(all_csvs) == 3:
+            data_testing, headers_testing = parseTrainingCSV(
+                all_csvs[2], train=train_mode
             )
+        assert (
+            headers_train == headers_testing
+        ), "The training and testing CSVs do not have the same header information."
 
         parameters = populate_header_in_parameters(parameters, headers_train)
         # if we are here, it is assumed that the user wants to do training
@@ -73,6 +82,7 @@ def main_run(data_csv, config_file, output_dir, train_mode, device, resume, rese
             TrainingManager_split(
                 dataframe_train=data_train,
                 dataframe_validation=data_validation,
+                dataframe_testing=data_testing,
                 outputDir=parameters["output_dir"],
                 parameters=parameters,
                 device=device,
diff --git a/GANDLF/training_manager.py b/GANDLF/training_manager.py
--- a/GANDLF/training_manager.py
+++ b/GANDLF/training_manager.py
@@ -303,7 +303,14 @@ def TrainingManager(dataframe, outputDir, parameters, device, resume, reset):
 
 
 def TrainingManager_split(
-    dataframe_train, dataframe_validation, outputDir, parameters, device, resume, reset
+    dataframe_train,
+    dataframe_validation,
+    dataframe_testing,
+    outputDir,
+    parameters,
+    device,
+    resume,
+    reset,
 ):
     """
     This is the training manager that ties all the training functionality together
@@ -311,6 +318,7 @@ def TrainingManager_split(
     Args:
         dataframe_train (pandas.DataFrame): The training data from CSV.
         dataframe_validation (pandas.DataFrame): The validation data from CSV.
+        dataframe_testing (pandas.DataFrame): The testing data from CSV.
         outputDir (str): The main output directory.
         parameters (dict): The parameters dictionary.
         device (str): The device to perform computations on.
@@ -336,5 +344,5 @@ def TrainingManager_split(
         output_dir=outputDir,
         device=device,
         params=parameters,
-        testing_data=None,
+        testing_data=dataframe_testing,
     )


Test patch:
diff --git a/testing/test_full.py b/testing/test_full.py
--- a/testing/test_full.py
+++ b/testing/test_full.py
@@ -1367,6 +1367,21 @@ def test_generic_cli_function_mainrun(device):
     )
     sanitize_outputDir()
 
+    with open(file_config_temp, "w") as file:
+        yaml.dump(parameters, file)
+
+    # testing train/valid/test split
+    main_run(
+        file_data + "," + file_data + "," + file_data,
+        file_config_temp,
+        outputDir,
+        True,
+        device,
+        resume=False,
+        reset=True,
+    )
+    sanitize_outputDir()
+
     print("passed")