Repository: Project-MONAI/MONAI

Issue:
modifying applied_operations when there are pending operations raises better error message
**Describe the bug**
When the metatensor has pending operations, modifying the `applied_operations` property may change the final transform output's `applied_operations` ordering.

for example, the following `LambdaD` pushes to the `applied_operations` while `pending_operations` is trying to combine multiple lazy operations:
```py
Compose(
    [
        ResizeWithPadOrCropD(keys=["img", "seg"], spatial_size=[80, 72, 80]),
        Rotated(keys=["img", "seg"], angle=[np.pi / 2, np.pi / 2, 0], mode="nearest", keep_size=False),
        Lambdad(keys=["img"], func=lambda x: torch.sigmoid((x - x.min()) / x.std())),
    ],
    lazy_evaluation=True,
)
```

this becomes an issue when we try to inverse the `applied_operations`.
test_auto3dseg_ensemble torch 1.9.1
**Describe the bug**
```
[2023-04-18T16:07:21.134Z] ======================================================================
[2023-04-18T16:07:21.134Z] ERROR: test_get_history (tests.test_auto3dseg_hpo.TestHPO)
[2023-04-18T16:07:21.134Z] ----------------------------------------------------------------------
[2023-04-18T16:07:21.134Z] Traceback (most recent call last):
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/tests/test_auto3dseg_hpo.py", line 134, in setUp
[2023-04-18T16:07:21.134Z]     bundle_generator.generate(work_dir, num_fold=1)
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/monai/apps/auto3dseg/bundle_gen.py", line 569, in generate
[2023-04-18T16:07:21.134Z]     gen_algo.export_to_disk(output_folder, name, fold=f_id)
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/monai/apps/auto3dseg/bundle_gen.py", line 160, in export_to_disk
[2023-04-18T16:07:21.134Z]     self.fill_records = self.fill_template_config(self.data_stats_files, self.output_path, **kwargs)
[2023-04-18T16:07:21.134Z]   File "/tmp/tmp7sdlj1x3/workdir/algorithm_templates/dints/scripts/algo.py", line 119, in fill_template_config
[2023-04-18T16:07:21.134Z]     mem = get_mem_from_visible_gpus()
[2023-04-18T16:07:21.134Z]   File "/tmp/tmp7sdlj1x3/workdir/algorithm_templates/dints/scripts/algo.py", line 31, in get_mem_from_visible_gpus
[2023-04-18T16:07:21.134Z]     available_mem_visible_gpus.append(torch.cuda.mem_get_info(device=d)[0])
[2023-04-18T16:07:21.134Z] AttributeError: module 'torch.cuda' has no attribute 'mem_get_info'
[2023-04-18T16:07:21.134Z] 
[2023-04-18T16:07:21.134Z] ======================================================================
[2023-04-18T16:07:21.134Z] ERROR: test_run_algo (tests.test_auto3dseg_hpo.TestHPO)
[2023-04-18T16:07:21.134Z] ----------------------------------------------------------------------
[2023-04-18T16:07:21.134Z] Traceback (most recent call last):
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/tests/test_auto3dseg_hpo.py", line 134, in setUp
[2023-04-18T16:07:21.134Z]     bundle_generator.generate(work_dir, num_fold=1)
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/monai/apps/auto3dseg/bundle_gen.py", line 569, in generate
[2023-04-18T16:07:21.134Z]     gen_algo.export_to_disk(output_folder, name, fold=f_id)
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/monai/apps/auto3dseg/bundle_gen.py", line 160, in export_to_disk
[2023-04-18T16:07:21.134Z]     self.fill_records = self.fill_template_config(self.data_stats_files, self.output_path, **kwargs)
[2023-04-18T16:07:21.134Z]   File "/tmp/tmp7sdlj1x3/workdir/algorithm_templates/dints/scripts/algo.py", line 119, in fill_template_config
[2023-04-18T16:07:21.134Z]     mem = get_mem_from_visible_gpus()
[2023-04-18T16:07:21.134Z]   File "/tmp/tmp7sdlj1x3/workdir/algorithm_templates/dints/scripts/algo.py", line 31, in get_mem_from_visible_gpus
[2023-04-18T16:07:21.134Z]     available_mem_visible_gpus.append(torch.cuda.mem_get_info(device=d)[0])
[2023-04-18T16:07:21.134Z] AttributeError: module 'torch.cuda' has no attribute 'mem_get_info'
[2023-04-18T16:07:21.134Z] 
[2023-04-18T16:07:21.134Z] ======================================================================
[2023-04-18T16:07:21.134Z] ERROR: test_run_optuna (tests.test_auto3dseg_hpo.TestHPO)
[2023-04-18T16:07:21.134Z] ----------------------------------------------------------------------
[2023-04-18T16:07:21.134Z] Traceback (most recent call last):
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/tests/test_auto3dseg_hpo.py", line 134, in setUp
[2023-04-18T16:07:21.134Z]     bundle_generator.generate(work_dir, num_fold=1)
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/monai/apps/auto3dseg/bundle_gen.py", line 569, in generate
[2023-04-18T16:07:21.134Z]     gen_algo.export_to_disk(output_folder, name, fold=f_id)
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/monai/apps/auto3dseg/bundle_gen.py", line 160, in export_to_disk
[2023-04-18T16:07:21.134Z]     self.fill_records = self.fill_template_config(self.data_stats_files, self.output_path, **kwargs)
[2023-04-18T16:07:21.134Z]   File "/tmp/tmp7sdlj1x3/workdir/algorithm_templates/dints/scripts/algo.py", line 119, in fill_template_config
[2023-04-18T16:07:21.134Z]     mem = get_mem_from_visible_gpus()
[2023-04-18T16:07:21.134Z]   File "/tmp/tmp7sdlj1x3/workdir/algorithm_templates/dints/scripts/algo.py", line 31, in get_mem_from_visible_gpus
[2023-04-18T16:07:21.134Z]     available_mem_visible_gpus.append(torch.cuda.mem_get_info(device=d)[0])
[2023-04-18T16:07:21.134Z] AttributeError: module 'torch.cuda' has no attribute 'mem_get_info'
[2023-04-18T16:07:21.134Z] 
[2023-04-18T16:07:21.134Z] ======================================================================
[2023-04-18T16:07:21.134Z] ERROR: test_ensemble (tests.test_auto3dseg_ensemble.TestEnsembleBuilder)
[2023-04-18T16:07:21.134Z] ----------------------------------------------------------------------
[2023-04-18T16:07:21.134Z] Traceback (most recent call last):
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/tests/test_auto3dseg_ensemble.py", line 140, in test_ensemble
[2023-04-18T16:07:21.134Z]     bundle_generator.generate(work_dir, num_fold=1)
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/monai/apps/auto3dseg/bundle_gen.py", line 569, in generate
[2023-04-18T16:07:21.134Z]     gen_algo.export_to_disk(output_folder, name, fold=f_id)
[2023-04-18T16:07:21.134Z]   File "/home/jenkins/agent/workspace/Monai-pytorch-versions/monai/apps/auto3dseg/bundle_gen.py", line 160, in export_to_disk
[2023-04-18T16:07:21.134Z]     self.fill_records = self.fill_template_config(self.data_stats_files, self.output_path, **kwargs)
[2023-04-18T16:07:21.134Z]   File "/tmp/tmp7sdlj1x3/workdir/algorithm_templates/dints/scripts/algo.py", line 119, in fill_template_config
[2023-04-18T16:07:21.134Z]     mem = get_mem_from_visible_gpus()
[2023-04-18T16:07:21.134Z]   File "/tmp/tmp7sdlj1x3/workdir/algorithm_templates/dints/scripts/algo.py", line 31, in get_mem_from_visible_gpus
[2023-04-18T16:07:21.134Z]     available_mem_visible_gpus.append(torch.cuda.mem_get_info(device=d)[0])
[2023-04-18T16:07:21.134Z] AttributeError: module 'torch.cuda' has no attribute 'mem_get_info'
[2023-04-18T16:07:21.134Z] 
[2023-04-18T16:07:21.134Z] ----------------------------------------------------------------------
```



Discussion:

Resolving patch:
diff --git a/monai/apps/auto3dseg/bundle_gen.py b/monai/apps/auto3dseg/bundle_gen.py
--- a/monai/apps/auto3dseg/bundle_gen.py
+++ b/monai/apps/auto3dseg/bundle_gen.py
@@ -36,7 +36,7 @@
 from monai.utils.enums import AlgoKeys
 
 logger = get_logger(module_name=__name__)
-ALGO_HASH = os.environ.get("MONAI_ALGO_HASH", "629ec7e")
+ALGO_HASH = os.environ.get("MONAI_ALGO_HASH", "5b4e379")
 
 __all__ = ["BundleAlgo", "BundleGen"]
 
diff --git a/monai/transforms/inverse.py b/monai/transforms/inverse.py
--- a/monai/transforms/inverse.py
+++ b/monai/transforms/inverse.py
@@ -221,6 +221,14 @@ def track_transform_meta(
                 info[LazyAttr.AFFINE] = convert_to_tensor(affine, device=torch.device("cpu"))
             out_obj.push_pending_operation(info)
         else:
+            if out_obj.pending_operations:
+                transform_name = info.get(TraceKeys.CLASS_NAME, "") if isinstance(info, dict) else ""
+                warnings.warn(
+                    f"Applying transform {transform_name} to a MetaTensor with pending operations "
+                    "is not supported (as this eventually changes the ordering of applied_operations when the pending "
+                    f"operations are executed). Please clear the pending operations before transform {transform_name}."
+                    f"\nPending operations: {[x.get(TraceKeys.CLASS_NAME) for x in out_obj.pending_operations]}."
+                )
             out_obj.push_applied_operation(info)
         if isinstance(data, Mapping):
             if not isinstance(data, dict):


Test patch:
diff --git a/tests/test_auto3dseg_ensemble.py b/tests/test_auto3dseg_ensemble.py
--- a/tests/test_auto3dseg_ensemble.py
+++ b/tests/test_auto3dseg_ensemble.py
@@ -76,7 +76,7 @@
 
 
 @skip_if_quick
-@SkipIfBeforePyTorchVersion((1, 9, 1))
+@SkipIfBeforePyTorchVersion((1, 11, 1))
 @unittest.skipIf(not has_tb, "no tensorboard summary writer")
 class TestEnsembleBuilder(unittest.TestCase):
     def setUp(self) -> None:
diff --git a/tests/test_auto3dseg_hpo.py b/tests/test_auto3dseg_hpo.py
--- a/tests/test_auto3dseg_hpo.py
+++ b/tests/test_auto3dseg_hpo.py
@@ -75,7 +75,7 @@ def skip_if_no_optuna(obj):
 }
 
 
-@SkipIfBeforePyTorchVersion((1, 9, 1))
+@SkipIfBeforePyTorchVersion((1, 11, 1))
 @unittest.skipIf(not has_tb, "no tensorboard summary writer")
 class TestHPO(unittest.TestCase):
     def setUp(self) -> None:
diff --git a/tests/test_integration_lazy_samples.py b/tests/test_integration_lazy_samples.py
--- a/tests/test_integration_lazy_samples.py
+++ b/tests/test_integration_lazy_samples.py
@@ -23,11 +23,15 @@
 
 import monai
 import monai.transforms as mt
-from monai.data import create_test_image_3d
+from monai.data import create_test_image_3d, decollate_batch
 from monai.utils import set_determinism
 from tests.utils import HAS_CUPY, DistTestCase, SkipIfBeforePyTorchVersion, skip_if_quick
 
 
+def _no_op(x):
+    return x
+
+
 def run_training_test(root_dir, device="cuda:0", cachedataset=0, readers=(None, None), num_workers=4, lazy=True):
     print(f"test case: {locals()}")
     images = sorted(glob(os.path.join(root_dir, "img*.nii.gz")))
@@ -64,6 +68,7 @@ def run_training_test(root_dir, device="cuda:0", cachedataset=0, readers=(None,
             ),
             mt.ResizeWithPadOrCropD(keys=["img", "seg"], spatial_size=[80, 72, 80]),
             mt.Rotated(keys=["img", "seg"], angle=[np.pi / 2, np.pi / 2, 0], mode="nearest", keep_size=False),
+            mt.Lambdad(keys=["img"], func=_no_op),
         ],
         lazy_evaluation=lazy,
         overrides=lazy_kwargs,
@@ -98,6 +103,9 @@ def run_training_test(root_dir, device="cuda:0", cachedataset=0, readers=(None,
         separate_folder=False,
         print_log=False,
     )
+    inverter = mt.Invertd(
+        keys="seg", orig_keys="img", transform=mt.Compose(train_transforms.transforms[-5:]), to_tensor=True
+    )
 
     # use batch_size=2 to load images and use RandCropByPosNegLabeld to generate 2 x 4 images for network training
     _g = torch.Generator()
@@ -110,9 +118,7 @@ def run_training_test(root_dir, device="cuda:0", cachedataset=0, readers=(None,
     for epoch in range(5):
         print("-" * 10)
         print(f"Epoch {epoch + 1}/5")
-        step = 0
-        for batch_data in train_loader:
-            step += 1
+        for step, batch_data in enumerate(train_loader, start=1):
             inputs, labels = batch_data["img"].to(device), batch_data["seg"].to(device)
             optimizer.zero_grad()
             outputs = model(inputs)
@@ -143,6 +149,16 @@ def run_training_test(root_dir, device="cuda:0", cachedataset=0, readers=(None,
                 saver(item)  # just testing the saving
                 saver(in_img)
                 saver(in_seg)
+    if lazy:
+        inverted = 0
+        try:
+            inverted = [inverter(b_data) for b_data in decollate_batch(batch_data)]
+        except RuntimeError as e:
+            if "Lambda" in str(e):
+                inverted = None
+        assert inverted is None, "invert LambdaD + lazy is not supported"
+    else:
+        [inverter(b_data) for b_data in decollate_batch(batch_data)]  # expecting no error
     return ops
 
 
diff --git a/tests/test_retinanet.py b/tests/test_retinanet.py
--- a/tests/test_retinanet.py
+++ b/tests/test_retinanet.py
@@ -183,7 +183,7 @@ def test_onnx(self, model, input_param, input_shape):
         data = torch.randn(input_shape)
         backbone = model(**input_param)
         if idx == 0:
-            test_onnx_save(backbone, data, rtol=1e-3, atol=0.1)
+            test_onnx_save(backbone, data, rtol=2e-3)
             return
         feature_extractor = resnet_fpn_feature_extractor(
             backbone=backbone,
@@ -193,7 +193,7 @@ def test_onnx(self, model, input_param, input_shape):
             returned_layers=[1, 2],
         )
         if idx == 1:
-            test_onnx_save(feature_extractor, data, rtol=1e-3)
+            test_onnx_save(feature_extractor, data, rtol=2e-3)
             return
         net = RetinaNet(
             spatial_dims=input_param["spatial_dims"],
@@ -203,7 +203,7 @@ def test_onnx(self, model, input_param, input_shape):
             size_divisible=32,
         )
         if idx == 2:
-            test_onnx_save(net, data, rtol=1e-3)
+            test_onnx_save(net, data, rtol=2e-3)
 
 
 if __name__ == "__main__":