Repository: mlcommons/GaNDLF

Issue:
Ensure memory pinning is happening properly
**Is your feature request related to a problem? Please describe.**
Currently, memory pinning doesn't work for CPU data

**Describe the solution you'd like**
Implement solution based on https://github.com/fepegar/torchio/discussions/568#discussioncomment-859027

**Describe alternatives you've considered**
N.A.

**Additional context**
N.A.



Discussion:

Resolving patch:
diff --git a/GANDLF/data/ImagesFromDataFrame.py b/GANDLF/data/ImagesFromDataFrame.py
--- a/GANDLF/data/ImagesFromDataFrame.py
+++ b/GANDLF/data/ImagesFromDataFrame.py
@@ -203,13 +203,11 @@ def ImagesFromDataFrame(dataframe,
             # sanity check for malformed csv
             if not os.path.isfile(str(dataframe[channel][patient])):
                 skip_subject = True
+
             # assigning the dict key to the channel
-            if not in_memory:
-                subject_dict[str(channel)] = Image(str(dataframe[channel][patient]), type=torchio.INTENSITY)
-            else:
-                img = sitk.ReadImage(str(dataframe[channel][patient]))
-                img_tensor = get_tensor_for_dataloader(img)
-                subject_dict[str(channel)] = Image(tensor=img_tensor, type=torchio.INTENSITY, path=dataframe[channel][patient])
+            img = sitk.ReadImage(str(dataframe[channel][patient]))
+            img_tensor = get_tensor_for_dataloader(img)
+            subject_dict[str(channel)] = Image(tensor=img_tensor, type=torchio.INTENSITY, path=dataframe[channel][patient])
             
             # if resize is requested, the perform per-image resize with appropriate interpolator
             if resize_images:
@@ -227,12 +225,10 @@ def ImagesFromDataFrame(dataframe,
         if labelHeader is not None:
             if not os.path.isfile(str(dataframe[labelHeader][patient])):
                 skip_subject = True
-            if not in_memory:
-                subject_dict['label'] = Image(str(dataframe[labelHeader][patient]), type=torchio.LABEL)
-            else:
-                img = sitk.ReadImage(str(dataframe[labelHeader][patient]))
-                img_tensor = get_tensor_for_dataloader(img)
-                subject_dict['label'] = Image(tensor=img_tensor, type=torchio.LABEL, path=dataframe[labelHeader][patient])
+                
+            img = sitk.ReadImage(str(dataframe[labelHeader][patient]))
+            img_tensor = get_tensor_for_dataloader(img)
+            subject_dict['label'] = Image(tensor=img_tensor, type=torchio.LABEL, path=dataframe[labelHeader][patient])
 
             # if resize is requested, the perform per-image resize with appropriate interpolator
             if resize_images:
@@ -261,9 +257,13 @@ def ImagesFromDataFrame(dataframe,
             # # padding image, but only for label sampler, because we don't want to pad for uniform
             if 'label' in sampler or 'weight' in sampler:
                 if enable_padding:
-                    psize_pad = list(np.asarray(np.ceil(np.divide(psize,2)), dtype=int))
+                    psize_pad = list(np.asarray(np.ceil(np.divide(patch_size, 2)), dtype=int))
                     padder = Pad(psize_pad, padding_mode = 'symmetric') # for modes: https://numpy.org/doc/stable/reference/generated/numpy.pad.html
                     subject = padder(subject)
+            
+            # load subject into memory: https://github.com/fepegar/torchio/discussions/568#discussioncomment-859027
+            if in_memory:
+                subject.load()
 
             # Appending this subject to the list of subjects
             subjects_list.append(subject)
diff --git a/GANDLF/parseConfig.py b/GANDLF/parseConfig.py
--- a/GANDLF/parseConfig.py
+++ b/GANDLF/parseConfig.py
@@ -1,22 +1,19 @@
 import sys, yaml, pkg_resources
 
-## dictionary to define defaults for appropriate options
+## dictionary to define defaults for appropriate options, which are evaluated
 parameter_defaults = {
     "weighted_loss": False, # whether weighted loss is to be used or not
     "verbose": False, # general application verbosity
     "q_verbose": False, # queue construction verbosity
     "medcam_enabled": False, # interpretability via medcam
     "save_output": False, # save outputs during validation/testing
-    "in_memory": False, # pin data to gpu memory
-    "patch_sampler": "uniform", # type of sampling strategy
+    "in_memory": False, # pin data to cpu memory
+    "pin_memory_dataloader": False, # pin data to gpu memory 
     "enable_padding": False, # if padding needs to be done when "patch_sampler" is "label"
     "scaling_factor": 1, # scaling factor for regression problems
-    "scheduler": "triangle_modified", # the default scheduler
-    "opt": "adam", # the optimizer
     "q_max_length": 100, # the max length of queue
     "q_samples_per_volume": 10, # number of samples per volume
     "q_num_workers": 4, # number of worker threads to use
-    "loss_function": "dc", # default loss
     "num_epochs":  100, # total number of epochs to train
     "patience":  100, # number of epochs to wait for performance improvement
     "batch_size": 1, # default batch size of training
@@ -24,12 +21,22 @@
     "learning_rate": 0.001, # default learning rate
 }
 
-def initialize_parameter(params, parameter_to_initialize, value = None):
+## dictionary to define string defaults for appropriate options
+parameter_defaults_string = {
+    "opt": "adam", # the optimizer
+    "patch_sampler": "uniform", # type of sampling strategy
+    "scheduler": "triangle_modified", # the default scheduler
+    "loss_function": "dc", # default loss
+}
+
+def initialize_parameter(params, parameter_to_initialize, value = None, evaluate = True):
     """
     Initializes the specified parameter with supplied value
     """
     if parameter_to_initialize in params:
-        pass
+        if evaluate:
+            if isinstance(params[parameter_to_initialize], str):
+                params[parameter_to_initialize] = eval(params[parameter_to_initialize])
     else:
         print("WARNING: Initializing \'" + parameter_to_initialize + "\' as " + str(value))
         params[parameter_to_initialize] = value
@@ -453,6 +460,9 @@ def parseConfig(config_file_path, version_check=True):
     params["parallel_compute_command"] = parallel_compute_command
 
     for current_parameter in parameter_defaults:
-        params = initialize_parameter(params, current_parameter, parameter_defaults[current_parameter])
+        params = initialize_parameter(params, current_parameter, parameter_defaults[current_parameter], True)
+    
+    for current_parameter in parameter_defaults_string:
+        params = initialize_parameter(params, current_parameter, parameter_defaults_string[current_parameter], False)
     
     return params
diff --git a/GANDLF/training_loop.py b/GANDLF/training_loop.py
--- a/GANDLF/training_loop.py
+++ b/GANDLF/training_loop.py
@@ -480,16 +480,16 @@ def training_loop(
         training_data_for_torch,
         batch_size=params["batch_size"],
         shuffle=True,
-        pin_memory=params["in_memory"],
+        pin_memory=False #params["pin_memory_dataloader"], # this is going OOM if True - needs investigation
     )
 
     val_dataloader = DataLoader(
-        validation_data_for_torch, batch_size=1, pin_memory=params["in_memory"]
+        validation_data_for_torch, batch_size=1, pin_memory=False #params["pin_memory_dataloader"], # this is going OOM if True - needs investigation
     )
 
     if testingDataDefined:
         test_dataloader = DataLoader(
-            test_data_for_torch, batch_size=1, pin_memory=params["in_memory"]
+            test_data_for_torch, batch_size=1, pin_memory=False #params["pin_memory_dataloader"], # this is going OOM if True - needs investigation
         )
 
     # Fetch the appropriate channel keys


Test patch:
diff --git a/testing/data.zip b/testing/data.zip
Binary files a/testing/data.zip and b/testing/data.zip differ
diff --git a/testing/test_full.py b/testing/test_full.py
--- a/testing/test_full.py
+++ b/testing/test_full.py
@@ -134,6 +134,7 @@ def test_train_segmentation_rad_3d(device):
   parameters['model']['dimension'] = 3
   parameters['model']['class_list'] = [0,1]
   parameters['model']['amp'] = True
+  parameters['in_memory'] = True
   parameters['model']['num_channels'] = len(parameters["headers"]["channelHeaders"])
   # loop through selected models and train for single epoch
   for model in all_models_segmentation: