Repository: Project-MONAI/MONAI

Issue:
 import cv2 + nvidia/pytorch:22.09-py3 + DistributedDataParallel. (FIND was unable to find an engine)
EDIT: the bug is reproducable in the newest nvidia/pytorch:22.09-py3 docker container, but is not reproducible in older container (older pytorch/cudnn)

Something in MetaTensor makes DistributedDataParallel fail 
(this is in addition to this bug https://github.com/Project-MONAI/MONAI/issues/5283)

For example this code fails

```
import torch.distributed as dist
import torch

from monai.data import MetaTensor
#from monai.config.type_definitions import NdarrayTensor

from torch.cuda.amp import autocast  
torch.autograd.set_detect_anomaly(True)

def main():

    ngpus_per_node = torch.cuda.device_count()
    torch.multiprocessing.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node,))

def main_worker(rank, ngpus_per_node):

    print(f"rank {rank}")

    dist.init_process_group(backend='nccl', init_method='tcp://127.0.0.1:23456', world_size=ngpus_per_node, rank=rank)
    torch.backends.cudnn.benchmark = True

    model = torch.nn.Conv3d(in_channels=1, out_channels=32, kernel_size=3, bias=True).to(rank)
    model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[rank], output_device=rank, find_unused_parameters=False)

    x = torch.ones(1, 1, 192, 192, 192).to(rank)
    with autocast(enabled=True):
        out = model(x)

    print("Done.", out.shape)

if __name__ == "__main__":
    main()
```

with error

```
-- Process 6 terminated with the following error:                                                                                                               
Traceback (most recent call last):                                                                                                                              
  File "/opt/conda/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 69, in _wrap                                                               
    fn(i, *args)                                                                                                                                                
  File "/mnt/amproj/Code/automl/tasks/hecktor22/autoconfig_segresnet/test_monai.py", line 29, in main_worker                                                    
    out = model(x)                                                                                                                                              
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1185, in _call_impl                                                            
    return forward_call(*input, **kwargs)                                                                                                                       
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1015, in forward                                                         
    output = self._run_ddp_forward(*inputs, **kwargs)                                                                                                           
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 976, in _run_ddp_forward                                                 
    return module_to_run(*inputs[0], **kwargs[0])                                                                                                               
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1185, in _call_impl                                                            
    return forward_call(*input, **kwargs)                                                                                                                       
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 613, in forward                                                                  
    return self._conv_forward(input, self.weight, self.bias)                                                                                                    
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 608, in _conv_forward
    return F.conv3d(
RuntimeError: FIND was unable to find an engine to execute this computation
```


The MetaTensor is actually never used/initialized here, but something it it (or it's imports) makes the code fail. Since we import MetaTensor everywhere, any code with it fails. I've traced it down to this import (inside of MetaTensor.py)
`from monai.config.type_definitions import NdarrayTensor`

importing this line also makes the code fail. 

Somehow it confuses conv3d operation, and possibly other operations


Discussion:

Resolving patch:
diff --git a/monai/__init__.py b/monai/__init__.py
--- a/monai/__init__.py
+++ b/monai/__init__.py
@@ -39,7 +39,17 @@
 
 # handlers_* have some external decorators the users may not have installed
 # *.so files and folder "_C" may not exist when the cpp extensions are not compiled
-excludes = "(^(monai.handlers))|(^(monai.bundle))|(^(monai.fl))|((\\.so)$)|(^(monai._C))|(.*(__main__)$)"
+excludes = "|".join(
+    [
+        "(^(monai.handlers))",
+        "(^(monai.bundle))",
+        "(^(monai.fl))",
+        "((\\.so)$)",
+        "(^(monai._C))",
+        "(.*(__main__)$)",
+        "(.*(video_dataset)$)",
+    ]
+)
 
 # load directory modules only, skip loading individual files
 load_submodules(sys.modules[__name__], False, exclude_pattern=excludes)
diff --git a/monai/data/__init__.py b/monai/data/__init__.py
--- a/monai/data/__init__.py
+++ b/monai/data/__init__.py
@@ -106,7 +106,9 @@
     worker_init_fn,
     zoom_affine,
 )
-from .video_dataset import CameraDataset, VideoDataset, VideoFileDataset
+
+# FIXME: workaround for https://github.com/Project-MONAI/MONAI/issues/5291
+# from .video_dataset import CameraDataset, VideoDataset, VideoFileDataset
 from .wsi_datasets import MaskedPatchWSIDataset, PatchWSIDataset, SlidingPatchWSIDataset
 from .wsi_reader import BaseWSIReader, CuCIMWSIReader, OpenSlideWSIReader, TiffFileWSIReader, WSIReader
 


Test patch:
diff --git a/tests/__init__.py b/tests/__init__.py
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -29,7 +29,7 @@ def _enter_pr_4800(self):
     return self
 
 
-# workaround for https://bugs.python.org/issue29620
+# FIXME: workaround for https://bugs.python.org/issue29620
 try:
     # Suppression for issue #494:  tests/__init__.py:34: error: Cannot assign to a method
     unittest.case._AssertWarnsContext.__enter__ = _enter_pr_4800  # type: ignore
diff --git a/tests/test_cv2_dist.py b/tests/test_cv2_dist.py
new file mode 100644
--- /dev/null
+++ b/tests/test_cv2_dist.py
@@ -0,0 +1,46 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import torch
+import torch.distributed as dist
+from torch.cuda.amp import autocast
+
+# FIXME: test for the workaround of https://github.com/Project-MONAI/MONAI/issues/5291
+from monai.config.deviceconfig import print_config
+from tests.utils import skip_if_no_cuda
+
+
+def main_worker(rank, ngpus_per_node):
+    dist.init_process_group(backend="nccl", init_method="tcp://127.0.0.1:12345", world_size=ngpus_per_node, rank=rank)
+    # `benchmark = True` is not compatible with openCV in PyTorch 22.09 docker for multi-gpu training
+    torch.backends.cudnn.benchmark = True
+
+    model = torch.nn.Conv3d(in_channels=1, out_channels=32, kernel_size=3, bias=True).to(rank)
+    model = torch.nn.parallel.DistributedDataParallel(
+        model, device_ids=[rank], output_device=rank, find_unused_parameters=False
+    )
+    x = torch.ones(1, 1, 192, 192, 192).to(rank)
+    with autocast(enabled=True):
+        model(x)
+
+
+@skip_if_no_cuda
+class TestCV2Dist(unittest.TestCase):
+    def test_cv2_cuda_ops(self):
+        print_config()
+        ngpus_per_node = torch.cuda.device_count()
+        torch.multiprocessing.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node,))
+
+
+if __name__ == "__main__":
+    unittest.main()