Repository: pytorch/ignite

Issue:
Length check fails for IterableDataset
I am using IterableDataset with unknown length. It fails inside the following `if` clause:

https://github.com/pytorch/ignite/blob/cc78b3bbd764da9ae7dee0df21676b2b8a50fba8/ignite/engine/engine.py#L642

Here is the logic why it fails: 
1. Data is PyTorch DataLoader, which has the `__len__` attr, so it goes into the `if` clause
2. It runs the code `epoch_length = len(data)`, and leads to `length = self._IterableDataset_len_called = len(self.dataset)` in `dataloader.py` in pytorch
3. Since dataset is an IterableDataset, it doesn't have length so it throws sth like `TypeError: object of type 'IterableDataset' has no len()`

My concern is that we shouldn't expect IterableDataset to have `__len__` and the code should not fail because of it. Any thoughts on this?


Discussion:
@snie2012 thanks for the report! Yes, you are right about that. We have a test with `IterableDataset` but with defined `epoch_length`: https://github.com/pytorch/ignite/blob/a2f302bf9eba2b04ba878766e2bd544ae5232950/tests/ignite/engine/test_engine.py#L511 Yes, definitely, we need to support `IterableDataset` with unknown length !

Resolving patch:
diff --git a/ignite/engine/deterministic.py b/ignite/engine/deterministic.py
--- a/ignite/engine/deterministic.py
+++ b/ignite/engine/deterministic.py
@@ -193,13 +193,7 @@ def _init_run(self) -> None:
             torch.backends.cudnn.benchmark = False
 
     def _setup_engine(self) -> None:
-        try:
-            self._dataloader_len = None
-            if hasattr(self.state.dataloader, "__len__"):
-                self._dataloader_len = len(self.state.dataloader)
-        except TypeError:
-            # _InfiniteConstantSampler can raise a TypeError on DataLoader length of a IterableDataset
-            self._dataloader_len = None
+        self._dataloader_len = self._get_data_length(self.state.dataloader)
 
         # if input data is torch dataloader we replace batch sampler by a batch sampler
         # such that its random sampling indices are reproducible by prefetching them before data iteration
diff --git a/ignite/engine/engine.py b/ignite/engine/engine.py
--- a/ignite/engine/engine.py
+++ b/ignite/engine/engine.py
@@ -639,10 +639,9 @@ def switch_batch(engine):
             if max_epochs is None:
                 max_epochs = 1
             if epoch_length is None:
-                if hasattr(data, "__len__"):
-                    epoch_length = len(data)
-                    if epoch_length < 1:
-                        raise ValueError("Input data has zero size. Please provide non-empty data")
+                epoch_length = self._get_data_length(data)
+                if epoch_length is not None and epoch_length < 1:
+                    raise ValueError("Input data has zero size. Please provide non-empty data")
 
             self.state.iteration = 0
             self.state.epoch = 0
@@ -664,6 +663,16 @@ def _init_timers(state: State):
         state.times[Events.EPOCH_COMPLETED.name] = 0.0
         state.times[Events.COMPLETED.name] = 0.0
 
+    def _get_data_length(self, data):
+        data_length = None
+        try:
+            if hasattr(data, "__len__"):
+                data_length = len(data)
+        except TypeError:
+            # _InfiniteConstantSampler can raise a TypeError on DataLoader length of a IterableDataset
+            pass
+        return data_length
+
     def _setup_engine(self) -> None:
         iteration = self.state.iteration
         self._dataloader_iter = iter(self.state.dataloader)


Test patch:
diff --git a/tests/ignite/engine/test_engine.py b/tests/ignite/engine/test_engine.py
--- a/tests/ignite/engine/test_engine.py
+++ b/tests/ignite/engine/test_engine.py
@@ -509,7 +509,10 @@ def test_multinode_distrib_gpu(distributed_context_multi_node_nccl):
 
 
 def test_engine_with_iterable_dataloader():
-    class MyIterableDataset(torch.utils.data.IterableDataset):
+
+    from torch.utils.data import DataLoader, IterableDataset, get_worker_info
+
+    class MyIterableDataset(IterableDataset):
         def __init__(self, start, end):
             super(MyIterableDataset).__init__()
             assert end > start, "this example code only works with end >= start"
@@ -519,19 +522,29 @@ def __init__(self, start, end):
         def __iter__(self):
             return iter(range(self.start, self.end))
 
-    ds = MyIterableDataset(0, 1000)
-    data_loader = torch.utils.data.DataLoader(ds, num_workers=2)
+    def _test(epoch_length=None):
 
-    counter = [0]
+        le = 50
+        num_workers = 4
+        ds = MyIterableDataset(0, le)
+        data_loader = DataLoader(ds, num_workers=num_workers)
 
-    def foo(e, b):
-        print("{}-{}: {}".format(e.state.epoch, e.state.iteration, b))
-        counter[0] += 1
+        counter = [0]
 
-    engine = Engine(foo)
-    engine.run(data_loader, epoch_length=10, max_epochs=5)
+        def foo(e, b):
+            print("{}-{}: {}".format(e.state.epoch, e.state.iteration, b))
+            counter[0] += 1
 
-    assert counter[0] == 50
+        engine = Engine(foo)
+        engine.run(data_loader, epoch_length=epoch_length, max_epochs=5)
+
+        epoch_length = le * num_workers if epoch_length is None else epoch_length
+        assert counter[0] == 5 * epoch_length
+
+    _test(epoch_length=20)
+
+    # tests issue : https://github.com/pytorch/ignite/issues/1076
+    _test(epoch_length=None)
 
 
 def test_engine_random_state():
@@ -621,7 +634,7 @@ def test_engine_with_dataloader_no_auto_batching():
     from torch.utils.data import DataLoader, BatchSampler, RandomSampler
 
     data = torch.rand(64, 4, 10)
-    data_loader = torch.utils.data.DataLoader(
+    data_loader = DataLoader(
         data, batch_size=None, sampler=BatchSampler(RandomSampler(data), batch_size=8, drop_last=True)
     )
 
@@ -871,14 +884,15 @@ def val_step(evaluator, batch):
 
 def test_set_data():
     # tests FR https://github.com/pytorch/ignite/issues/833
+    from torch.utils.data import DataLoader
 
     num_iters1 = 10
     num_iters2 = 20
     batch_size = 4
 
     torch.manual_seed(1)
-    data1 = torch.utils.data.DataLoader(torch.rand(num_iters1 * batch_size, 11), batch_size=batch_size)
-    data2 = torch.utils.data.DataLoader(torch.rand(num_iters2 * batch_size, 22), batch_size=batch_size)
+    data1 = DataLoader(torch.rand(num_iters1 * batch_size, 11), batch_size=batch_size)
+    data2 = DataLoader(torch.rand(num_iters2 * batch_size, 22), batch_size=batch_size)
 
     switch_iteration = 35