Repository: Project-MONAI/MONAI

Issue:
MONAI Workflow incompatible with ThreadBuffer
**Describe the bug**
ThreadBuffer as now coded wraps a DataLoader thus it would be logical for Workflow class to accept a ThreadBuffer as the source of data but it is not allowed.

**To Reproduce**
Instantiate a Workflow class to which you give a ThreadBuffer wrapped DataLoader as a dataloader.

**Expected behavior**
Just for the workflow to work as it is with a dataloader.

**Additional context**
https://github.com/Project-MONAI/MONAI/blob/master/monai/engines/workflow.py#L93



Discussion:

Resolving patch:
diff --git a/monai/engines/evaluator.py b/monai/engines/evaluator.py
--- a/monai/engines/evaluator.py
+++ b/monai/engines/evaluator.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Callable, Dict, Optional, Sequence, Tuple
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, Optional, Sequence, Tuple, Union
 
 import torch
 from torch.utils.data import DataLoader
@@ -38,7 +38,7 @@ class Evaluator(Workflow):
 
     Args:
         device: an object representing the device on which to run.
-        val_data_loader: Ignite engine use data_loader to run, must be torch.DataLoader.
+        val_data_loader: Ignite engine use data_loader to run, must be Iterable or torch.DataLoader.
         epoch_length: number of iterations for one epoch, default to `len(val_data_loader)`.
         non_blocking: if True and this copy is between CPU and GPU, the copy may occur asynchronously
             with respect to the host. For other cases, this argument has no effect.
@@ -60,7 +60,7 @@ class Evaluator(Workflow):
     def __init__(
         self,
         device: torch.device,
-        val_data_loader: DataLoader,
+        val_data_loader: Union[Iterable, DataLoader],
         epoch_length: Optional[int] = None,
         non_blocking: bool = False,
         prepare_batch: Callable = default_prepare_batch,
@@ -110,7 +110,7 @@ class SupervisedEvaluator(Evaluator):
 
     Args:
         device: an object representing the device on which to run.
-        val_data_loader: Ignite engine use data_loader to run, must be torch.DataLoader.
+        val_data_loader: Ignite engine use data_loader to run, must be Iterable, typically be torch.DataLoader.
         network: use the network to run model forward.
         epoch_length: number of iterations for one epoch, default to `len(val_data_loader)`.
         non_blocking: if True and this copy is between CPU and GPU, the copy may occur asynchronously
@@ -134,7 +134,7 @@ class SupervisedEvaluator(Evaluator):
     def __init__(
         self,
         device: torch.device,
-        val_data_loader: DataLoader,
+        val_data_loader: Union[Iterable, DataLoader],
         network: torch.nn.Module,
         epoch_length: Optional[int] = None,
         non_blocking: bool = False,
@@ -215,7 +215,7 @@ class EnsembleEvaluator(Evaluator):
 
     Args:
         device: an object representing the device on which to run.
-        val_data_loader: Ignite engine use data_loader to run, must be torch.DataLoader.
+        val_data_loader: Ignite engine use data_loader to run, must be Iterable, typically be torch.DataLoader.
         epoch_length: number of iterations for one epoch, default to `len(val_data_loader)`.
         networks: use the networks to run model forward in order.
         pred_keys: the keys to store every prediction data.
@@ -241,7 +241,7 @@ class EnsembleEvaluator(Evaluator):
     def __init__(
         self,
         device: torch.device,
-        val_data_loader: DataLoader,
+        val_data_loader: Union[Iterable, DataLoader],
         networks: Sequence[torch.nn.Module],
         pred_keys: Sequence[str],
         epoch_length: Optional[int] = None,
diff --git a/monai/engines/trainer.py b/monai/engines/trainer.py
--- a/monai/engines/trainer.py
+++ b/monai/engines/trainer.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Callable, Dict, Optional, Sequence, Tuple, Union
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, Optional, Sequence, Tuple, Union
 
 import torch
 from torch.optim.optimizer import Optimizer
@@ -58,7 +58,7 @@ class SupervisedTrainer(Trainer):
     Args:
         device: an object representing the device on which to run.
         max_epochs: the total epoch number for trainer to run.
-        train_data_loader: Ignite engine use data_loader to run, must be torch.DataLoader.
+        train_data_loader: Ignite engine use data_loader to run, must be Iterable or torch.DataLoader.
         network: to train with this network.
         optimizer: the optimizer associated to the network.
         loss_function: the loss function associated to the optimizer.
@@ -85,7 +85,7 @@ def __init__(
         self,
         device: torch.device,
         max_epochs: int,
-        train_data_loader: DataLoader,
+        train_data_loader: Union[Iterable, DataLoader],
         network: torch.nn.Module,
         optimizer: Optimizer,
         loss_function: Callable,
@@ -251,6 +251,9 @@ def __init__(
         additional_metrics: Optional[Dict[str, Metric]] = None,
         train_handlers: Optional[Sequence] = None,
     ):
+        if not isinstance(train_data_loader, DataLoader):
+            raise ValueError("train_data_loader must be PyTorch DataLoader.")
+
         # set up Ignite engine and environments
         super().__init__(
             device=device,
@@ -296,7 +299,7 @@ def _iteration(
             raise ValueError("must provide batch data for current iteration.")
 
         d_input = self.prepare_batch(batchdata, engine.state.device, engine.non_blocking)
-        batch_size = self.data_loader.batch_size
+        batch_size = self.data_loader.batch_size  # type: ignore
         g_input = self.g_prepare_batch(batch_size, self.latent_shape, engine.state.device, engine.non_blocking)
         g_output = self.g_inferer(g_input, self.g_network)
 
diff --git a/monai/engines/workflow.py b/monai/engines/workflow.py
--- a/monai/engines/workflow.py
+++ b/monai/engines/workflow.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Callable, Dict, Optional, Sequence
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, Optional, Sequence, Union
 
 import torch
 import torch.distributed as dist
@@ -44,7 +44,7 @@ class Workflow(IgniteEngine):  # type: ignore[valid-type, misc] # due to optiona
     Args:
         device: an object representing the device on which to run.
         max_epochs: the total epoch number for engine to run, validator and evaluator have only 1 epoch.
-        data_loader: Ignite engine use data_loader to run, must be torch.DataLoader.
+        data_loader: Ignite engine use data_loader to run, must be Iterable or torch.DataLoader.
         epoch_length: number of iterations for one epoch, default to `len(data_loader)`.
         non_blocking: if True and this copy is between CPU and GPU, the copy may occur asynchronously
             with respect to the host. For other cases, this argument has no effect.
@@ -73,7 +73,7 @@ def __init__(
         self,
         device: torch.device,
         max_epochs: int,
-        data_loader: DataLoader,
+        data_loader: Union[Iterable, DataLoader],
         epoch_length: Optional[int] = None,
         non_blocking: bool = False,
         prepare_batch: Callable = default_prepare_batch,
@@ -90,14 +90,20 @@ def __init__(
             super().__init__(self._iteration)
         if not isinstance(device, torch.device):
             raise TypeError(f"device must be a torch.device but is {type(device).__name__}.")
-        if not isinstance(data_loader, DataLoader):
-            raise TypeError(f"data_loader must be a torch.utils.data.DataLoader but is {type(data_loader).__name__}.")
-        sampler = data_loader.__dict__["sampler"]
-        if isinstance(sampler, DistributedSampler):
 
-            @self.on(Events.EPOCH_STARTED)
-            def set_sampler_epoch(engine: Engine):
-                sampler.set_epoch(engine.state.epoch)
+        if isinstance(data_loader, DataLoader):
+            sampler = data_loader.__dict__["sampler"]
+            if isinstance(sampler, DistributedSampler):
+
+                @self.on(Events.EPOCH_STARTED)
+                def set_sampler_epoch(engine: Engine):
+                    sampler.set_epoch(engine.state.epoch)
+
+            if epoch_length is None:
+                epoch_length = len(data_loader)
+        else:
+            if epoch_length is None:
+                raise ValueError("if data_loader is not PyTorch DataLoader, must specify the epoch_length.")
 
         # set all sharable data for the workflow based on Ignite engine.state
         self.state = State(
@@ -106,7 +112,7 @@ def set_sampler_epoch(engine: Engine):
             iteration=0,
             epoch=0,
             max_epochs=max_epochs,
-            epoch_length=len(data_loader) if epoch_length is None else epoch_length,
+            epoch_length=epoch_length,
             output=None,
             batch=None,
             metrics={},


Test patch: