Repository: mlcommons/GaNDLF

Issue:
Preprocess is not functioning in certain conditions
**Describe the bug**
When `'path'` is absent from the `subject` dictionary in the first image of a subject, the entire process fails because `subject["1"]["path"][0]` is empty.

**To Reproduce**
Steps to reproduce the behavior:
1. Put a case where `subject["1"]["path"][0]` is empty.
2. Run `gandlf_preprocess` on it using any config.
3. See error

**Expected behavior**
Even if the `'path'` is absent, it should get picked up from the `'path_to_metadata'` key.

**Screenshots**
N.A.

**GaNDLF Version**
<!-- Put the output of the following command:
python -c 'import GANDLF as g;print(g.__version__)'
-->
0.0.16-dev

**Desktop (please complete the following information):**
N.A.

**Additional context**
N.A.


Discussion:

Resolving patch:
diff --git a/GANDLF/cli/preprocess_and_save.py b/GANDLF/cli/preprocess_and_save.py
--- a/GANDLF/cli/preprocess_and_save.py
+++ b/GANDLF/cli/preprocess_and_save.py
@@ -69,11 +69,6 @@ def preprocess_and_save(
     base_df = get_dataframe(data_csv)
     # ensure csv only contains lower case columns
     base_df.columns = base_df.columns.str.lower()
-    # only store the column names
-    output_columns_to_write = base_df.to_dict()
-    for key in output_columns_to_write.keys():
-        output_columns_to_write[key] = []
-
     # keep a record of the keys which contains only images
     keys_with_images = parameters["headers"]["channelHeaders"]
     keys_with_images = [str(x) for x in keys_with_images]
@@ -99,11 +94,11 @@ def preprocess_and_save(
         tqdm(dataloader_for_processing, desc="Looping over data")
     ):
         # initialize the current_output_dir
-        current_output_dir = os.path.join(output_dir, str(subject["subject_id"][0]))
+        current_output_dir = os.path.abspath(
+            os.path.join(output_dir, str(subject["subject_id"][0]))
+        )
         Path(current_output_dir).mkdir(parents=True, exist_ok=True)
 
-        output_columns_to_write["subjectid"].append(subject["subject_id"][0])
-
         subject_dict_to_write, subject_process = {}, {}
 
         # start constructing the torchio.Subject object
@@ -154,29 +149,27 @@ def preprocess_and_save(
                     subject_dict_to_write["label"] = patch["label"]
 
         # write new images
-        common_ext = get_filename_extension_sanitized(subject["1"]["path"][0])
+        common_ext = get_filename_extension_sanitized(subject["path_to_metadata"][0])
         # in cases where the original image has a file format that does not support
         # RGB floats, use the "vtk" format
         if common_ext in [".png", ".jpg", ".jpeg", ".bmp", ".tiff", ".tif"]:
             common_ext = ".vtk"
 
-        if subject["1"]["path"][0] != "":
-            image_for_info_copy = sitk.ReadImage(subject["1"]["path"][0])
-        else:
-            image_for_info_copy = subject_dict_to_write["1"].as_sitk()
-        correct_spacing_for_info_copy = subject["spacing"][0].tolist()
-        for channel in parameters["headers"]["channelHeaders"]:
+        image_for_info_copy = subject_dict_to_write[
+            str(parameters["headers"]["channelHeaders"][0])
+        ].as_sitk()
+        for index, channel in enumerate(parameters["headers"]["channelHeaders"]):
             image_file = Path(
                 os.path.join(
                     current_output_dir,
-                    subject["subject_id"][0] + "_" + str(channel) + common_ext,
+                    subject["subject_id"][0] + "_" + str(index) + common_ext,
                 )
             ).as_posix()
-            output_columns_to_write["channel_" + str(channel - 1)].append(image_file)
+            base_df["channel_" + str(index)] = image_file
             image_to_write = subject_dict_to_write[str(channel)].as_sitk()
             image_to_write.SetOrigin(image_for_info_copy.GetOrigin())
             image_to_write.SetDirection(image_for_info_copy.GetDirection())
-            image_to_write.SetSpacing(correct_spacing_for_info_copy)
+            image_to_write.SetSpacing(image_for_info_copy.GetSpacing())
             if not os.path.isfile(image_file):
                 try:
                     sitk.WriteImage(image_to_write, image_file)
@@ -194,11 +187,11 @@ def preprocess_and_save(
                     current_output_dir, subject["subject_id"][0] + "_label" + common_ext
                 )
             ).as_posix()
-            output_columns_to_write["label"].append(image_file)
+            base_df["label"] = image_file
             image_to_write = subject_dict_to_write["label"].as_sitk()
             image_to_write.SetOrigin(image_for_info_copy.GetOrigin())
             image_to_write.SetDirection(image_for_info_copy.GetDirection())
-            image_to_write.SetSpacing(correct_spacing_for_info_copy)
+            image_to_write.SetSpacing(image_for_info_copy.GetSpacing())
             if not os.path.isfile(image_file):
                 try:
                     sitk.WriteImage(image_to_write, image_file)
@@ -212,16 +205,12 @@ def preprocess_and_save(
         # ensure prediction headers are getting saved, as well
         if len(parameters["headers"]["predictionHeaders"]) > 1:
             for key in parameters["headers"]["predictionHeaders"]:
-                output_columns_to_write["valuetopredict_" + str(key)].append(
-                    str(subject["value_" + str(key)].numpy()[0])
+                base_df["valuetopredict_" + str(key)] = str(
+                    subject["value_" + str(key)].numpy()[0]
                 )
         elif len(parameters["headers"]["predictionHeaders"]) == 1:
-            output_columns_to_write["valuetopredict"].append(
-                str(subject["value_0"].numpy()[0])
-            )
+            base_df["valuetopredict"] = str(subject["value_0"].numpy()[0])
 
     path_for_csv = Path(os.path.join(output_dir, "data_processed.csv")).as_posix()
     print("Writing final csv for subsequent training: ", path_for_csv)
-    pd.DataFrame.from_dict(data=output_columns_to_write).to_csv(
-        path_for_csv, header=True, index=False
-    )
+    base_df.to_csv(path_for_csv, header=True, index=False)


Test patch:
diff --git a/testing/test_full.py b/testing/test_full.py
--- a/testing/test_full.py
+++ b/testing/test_full.py
@@ -1232,6 +1232,15 @@ def test_generic_cli_function_preprocess():
     sanitize_outputDir()
     file_config_temp = get_temp_config_path()
     file_data = os.path.join(inputDir, "train_2d_rad_segmentation.csv")
+    input_data_df = pd.read_csv(file_data)
+    # add random metadata to ensure it gets preserved
+    input_data_df["metadata_test_string"] = input_data_df.shape[0] * ["test"]
+    input_data_df["metadata_test_float"] = np.random.rand(input_data_df.shape[0])
+    input_data_df["metadata_test_int"] = np.random.randint(
+        0, 100, input_data_df.shape[0]
+    )
+    temp_csv = os.path.join(outputDir, "temp.csv")
+    input_data_df.to_csv(temp_csv)
 
     parameters = parseConfig(file_config)
     parameters["modality"] = "rad"
@@ -1254,13 +1263,61 @@ def test_generic_cli_function_preprocess():
     with open(file_config_temp, "w") as outfile:
         yaml.dump(parameters, outfile, default_flow_style=False)
 
-    preprocess_and_save(file_data, file_config_temp, outputDir)
+    preprocess_and_save(temp_csv, file_config_temp, outputDir)
+    training_data, parameters["headers"] = parseTrainingCSV(
+        outputDir + "/data_processed.csv"
+    )
+
+    # check that the length of training data is what we expect
+    assert (
+        len(training_data) == input_data_df.shape[0]
+    ), "Number of subjects in dataframe is not same as that of input dataframe"
+    assert (
+        len(training_data.columns) == len(input_data_df.columns) + 1
+    ), "Number of columns in output dataframe is not same as that of input dataframe"  # the +1 is for the added index column
+    sanitize_outputDir()
+
+    ## regression/classification preprocess
+    file_config = os.path.join(testingDir, "config_regression.yaml")
+    parameters = parseConfig(file_config)
+    parameters["modality"] = "rad"
+    parameters["patch_size"] = patch_size["2D"]
+    parameters["model"]["dimension"] = 2
+    parameters["model"]["amp"] = False
+    # read and parse csv
+    parameters["model"]["num_channels"] = 3
+    parameters["scaling_factor"] = 1
+    parameters["model"]["onnx_export"] = False
+    parameters["model"]["print_summary"] = False
+    parameters["data_preprocessing"]["to_canonical"] = None
+    parameters["data_preprocessing"]["rgba_to_rgb"] = None
+    file_data = os.path.join(inputDir, "train_2d_rad_regression.csv")
+    input_data_df = pd.read_csv(file_data)
+    # add random metadata to ensure it gets preserved
+    input_data_df["metadata_test_string"] = input_data_df.shape[0] * ["test"]
+    input_data_df["metadata_test_float"] = np.random.rand(input_data_df.shape[0])
+    input_data_df["metadata_test_int"] = np.random.randint(
+        0, 100, input_data_df.shape[0]
+    )
+    input_data_df.to_csv(temp_csv)
+
+    # store this separately for preprocess testing
+    with open(file_config_temp, "w") as outfile:
+        yaml.dump(parameters, outfile, default_flow_style=False)
+
+    preprocess_and_save(temp_csv, file_config_temp, outputDir)
     training_data, parameters["headers"] = parseTrainingCSV(
         outputDir + "/data_processed.csv"
     )
 
     # check that the length of training data is what we expect
-    assert len(training_data) == 10, "Number of rows in dataframe is not 10"
+    assert (
+        len(training_data) == input_data_df.shape[0]
+    ), "Number of subjects in dataframe is not same as that of input dataframe"
+    assert (
+        len(training_data.columns) == len(input_data_df.columns) + 1
+    ), "Number of columns in output dataframe is not same as that of input dataframe"  # the +1 is for the added index column
+    sanitize_outputDir()
 
     print("passed")
 
@@ -2421,19 +2478,20 @@ def test_train_segmentation_unet_conversion_rad_3d(device):
     parameters = populate_header_in_parameters(parameters, parameters["headers"])
     # loop through selected models and train for single epoch
     for model in ["unet", "unet_multilayer", "lightunet_multilayer"]:
-        parameters["model"]["converter_type"] = random.choice(["acs", "soft", "conv3d"])
-        parameters["model"]["architecture"] = model
-        parameters["nested_training"]["testing"] = -5
-        parameters["nested_training"]["validation"] = -5
-        sanitize_outputDir()
-        TrainingManager(
-            dataframe=training_data,
-            outputDir=outputDir,
-            parameters=parameters,
-            device=device,
-            resume=False,
-            reset=True,
-        )
+        for converter_type in ["acs", "soft", "conv3d"]:
+            parameters["model"]["converter_type"] = converter_type
+            parameters["model"]["architecture"] = model
+            parameters["nested_training"]["testing"] = -5
+            parameters["nested_training"]["validation"] = -5
+            sanitize_outputDir()
+            TrainingManager(
+                dataframe=training_data,
+                outputDir=outputDir,
+                parameters=parameters,
+                device=device,
+                resume=False,
+                reset=True,
+            )
 
     print("passed")