Repository: mlcommons/GaNDLF

Issue:
Allow classification inference from different architectures to be combined
**Is your feature request related to a problem? Please describe.**
Currently, classification results from multiple folds can be combined, which is fine but this does not allow outputs from multiple architectures to get combined.

**Describe the solution you'd like**
See title.

**Describe alternatives you've considered**
Perform inference and combine outside GaNDLF.

**Additional context**
N.A.



Discussion:

Resolving patch:
diff --git a/GANDLF/inference_manager.py b/GANDLF/inference_manager.py
--- a/GANDLF/inference_manager.py
+++ b/GANDLF/inference_manager.py
@@ -22,64 +22,78 @@ def InferenceManager(dataframe, outputDir, parameters, device):
 
     n_folds = parameters["nested_training"]["validation"]
 
-    fold_dirs = []
-    if n_folds > 1:
-        directories = sorted(os.listdir(outputDir))
-        for d in directories:
-            if d.isdigit():
-                fold_dirs.append(os.path.join(outputDir, d, ""))
-    else:
-        fold_dirs = [outputDir]
-
-    # this is for the case where inference is happening using a single model
-    if len(fold_dirs) == 0:
-        fold_dirs = [outputDir]
-
-    probs_list = []
-    class_list = None
-    is_classification = parameters["problem_type"] == "classification"
-
-    # initialize model type for processing: if not defined, default to torch
-    if not ("type" in parameters["model"]):
-        parameters["model"]["type"] = "torch"
-
-    for fold_dir in fold_dirs:
-        parameters["current_fold_dir"] = fold_dir
-        inference_loop(
-            inferenceDataFromPickle=inferenceData_full,
-            outputDir_or_optimizedModel=fold_dir,
-            device=device,
-            parameters=parameters,
-        )
+    outputDir_split = [outputDir]
+    if "," in outputDir:
+        outputDir_split = outputDir.split(",")
+
+    averaged_probs_list = []
+    for current_outputDir in outputDir_split:
+        fold_dirs = []
+        if n_folds > 1:
+            directories = sorted(os.listdir(current_outputDir))
+            for d in directories:
+                if d.isdigit():
+                    fold_dirs.append(os.path.join(current_outputDir, d, ""))
+        else:
+            fold_dirs = [current_outputDir]
+
+        # this is for the case where inference is happening using a single model
+        if len(fold_dirs) == 0:
+            fold_dirs = [current_outputDir]
 
-        if parameters["problem_type"] == "classification":
-            logits_dir = os.path.join(fold_dir, "logits.csv")
-            is_logits_dir_exist = os.path.isfile(logits_dir)
+        probs_list = []
+        class_list = None
+        is_classification = parameters["problem_type"] == "classification"
 
-            if is_classification and is_logits_dir_exist:
-                # fold_logits = np.genfromtxt(logits_dir, delimiter=",")
-                class_list = [str(c) for c in parameters["model"]["class_list"]]
-                fold_logits = pd.read_csv(logits_dir)[class_list].values
-                fold_logits = torch.from_numpy(fold_logits)
-                fold_probs = F.softmax(fold_logits, dim=1)
-                probs_list.append(fold_probs)
+        # initialize model type for processing: if not defined, default to torch
+        if not ("type" in parameters["model"]):
+            parameters["model"]["type"] = "torch"
+
+        for fold_dir in fold_dirs:
+            parameters["current_fold_dir"] = fold_dir
+            inference_loop(
+                inferenceDataFromPickle=inferenceData_full,
+                outputDir_or_optimizedModel=fold_dir,
+                device=device,
+                parameters=parameters,
+            )
+
+            if is_classification:
+                logits_dir = os.path.join(fold_dir, "logits.csv")
+                is_logits_dir_exist = os.path.isfile(logits_dir)
+
+                if is_logits_dir_exist:
+                    # fold_logits = np.genfromtxt(logits_dir, delimiter=",")
+                    class_list = [str(c) for c in parameters["model"]["class_list"]]
+                    fold_logits = pd.read_csv(logits_dir)[class_list].values
+                    fold_logits = torch.from_numpy(fold_logits)
+                    fold_probs = F.softmax(fold_logits, dim=1)
+                    probs_list.append(fold_probs)
+
+        if is_classification and (n_folds > 1):
+            probs_list = torch.stack(probs_list)
+            averaged_probs_list.append(torch.mean(probs_list, 0))
 
     # this logic should be changed if we want to do multi-fold inference for histo images
-    if (parameters["modality"] == "rad") and probs_list and is_classification:
+    if (parameters["modality"] == "rad") and averaged_probs_list and is_classification:
         columns = ["SubjectID", "PredictedClass"] + parameters["model"]["class_list"]
         averaged_probs_df = pd.DataFrame(columns=columns)
         averaged_probs_df.SubjectID = dataframe[0]
 
-        probs_list = torch.stack(probs_list)
-        averaged_probs = torch.mean(probs_list, 0).numpy()
-        averaged_probs_df[class_list] = averaged_probs
+        averaged_probs_across_models = torch.stack(averaged_probs_list)
+        averaged_probs_across_models = torch.mean(
+            averaged_probs_across_models, 0
+        ).numpy()
+        averaged_probs_df[class_list] = averaged_probs_across_models
         averaged_probs_df.PredictedClass = [
-            class_list[a] for a in averaged_probs.argmax(1)
+            class_list[a] for a in averaged_probs_across_models.argmax(1)
         ]
-        filepath_to_save = os.path.join(outputDir, "final_preds_and_avg_probs.csv")
+        filepath_to_save = os.path.join(
+            outputDir_split[0], "final_preds_and_avg_probs.csv"
+        )
         if os.path.isfile(filepath_to_save):
             filepath_to_save = os.path.join(
-                outputDir,
+                outputDir_split[0],
                 "final_preds_and_avg_probs" + get_unique_timestamp() + ".csv",
             )
         averaged_probs_df.to_csv(filepath_to_save, index=False)


Test patch:
diff --git a/testing/test_full.py b/testing/test_full.py
--- a/testing/test_full.py
+++ b/testing/test_full.py
@@ -845,7 +845,7 @@ def test_train_inference_classification_with_logits_multiple_folds_rad_3d(device
     parameters["output_dir"] = outputDir  # this is in inference mode
     InferenceManager(
         dataframe=training_data,
-        outputDir=outputDir,
+        outputDir=outputDir + "," + outputDir,
         parameters=parameters,
         device=device,
     )