Repository: pytorch/ignite

Issue:
Auto dataloader - ValueError: batch_size should be a positive integer value, but got batch_size=0
## 🐛 Bug description

The following code will brake if `batch_size` is smaller than world size :
```python
import torch
import ignite.distributed as idist

def foo(i):
    data = torch.arange(100).reshape(25, 4)
    data_loader = idist.auto_dataloader(data, batch_size=6, num_workers=12)


if __name__ == "__main__":
    idist.spawn("gloo", foo, args=(), nproc_per_node=8)    
```
```
ValueError: batch_size should be a positive integer value, but got batch_size=0
```

A fix can be:
- keep batch size as provided if smaller than world size
- same for `num_workers`

## Environment

 - PyTorch Version (e.g., 1.4): 1.5.0
 - Ignite Version (e.g., 0.3.0): master
 - OS (e.g., Linux): linux
 - How you installed Ignite (`conda`, `pip`, source):
 - Python version: 3.7
 - Any other relevant information:


@InCogNiTo124 would you like to fix this as you recently played around `auto_*` ?


Discussion:
Sure, I will look into it sometime this week. Is there a deadline I should know of?
@InCogNiTo124 thanks! I'd say sooner is better. Let's say to have it done before this friday. Anyway, no worries about if you haven't time for that. If it becomes urgent, we'll do it :)
@vfdev-5 alright, I'll let you know if I can't fix it by then :)
I think I know the source of the bug, the `world_size` in this example is `8`, but `batch_size` is `6`, so `ignite/distributed/auto.py:L68` makes it `0`.


> A fix **cant** be:

@vfdev-5 I'm not sure if this is a typo or not
@InCogNiTo124 yes, you are correct about that: 6 // 8 = 0

> >   A fix cant be:

> @vfdev-5 I'm not sure if this is a typo or not

And yes, it is a typo. I fixed it.

Resolving patch:
diff --git a/ignite/distributed/auto.py b/ignite/distributed/auto.py
--- a/ignite/distributed/auto.py
+++ b/ignite/distributed/auto.py
@@ -21,8 +21,8 @@ def auto_dataloader(dataset, **kwargs):
 
     Internally, we create a dataloader with provided kwargs while applying the following updates:
 
-    - batch size is scaled by world size: ``batch_size / world_size``.
-    - number of workers is scaled by number of local processes: ``num_workers / nprocs``.
+    - batch size is scaled by world size: ``batch_size / world_size`` if larger or equal world size.
+    - number of workers is scaled by number of local processes: ``num_workers / nprocs`` if larger or equal world size.
     - if no sampler provided by user, `torch DistributedSampler` is setup.
     - if a sampler is provided by user, it is wrapped by :class:`~ignite.distributed.auto.DistributedProxySampler`.
     - if the default device is 'cuda', `pin_memory` is automatically set to `True`.
@@ -63,13 +63,12 @@ def auto_dataloader(dataset, **kwargs):
     world_size = idist.get_world_size()
 
     logger = setup_logger(__name__ + ".auto_dataloader")
-
     if world_size > 1:
-        if "batch_size" in kwargs:
+        if "batch_size" in kwargs and kwargs["batch_size"] >= world_size:
             kwargs["batch_size"] //= world_size
 
-        if "num_workers" in kwargs:
-            nproc = idist.get_nproc_per_node()
+        nproc = idist.get_nproc_per_node()
+        if "num_workers" in kwargs and kwargs["num_workers"] >= nproc:
             kwargs["num_workers"] = (kwargs["num_workers"] + nproc - 1) // nproc
 
         if "batch_sampler" not in kwargs:


Test patch:
diff --git a/tests/ignite/distributed/test_auto.py b/tests/ignite/distributed/test_auto.py
--- a/tests/ignite/distributed/test_auto.py
+++ b/tests/ignite/distributed/test_auto.py
@@ -12,7 +12,7 @@
 from ignite.distributed.auto import DistributedProxySampler, auto_dataloader, auto_model, auto_optim
 
 
-def _test_auto_dataloader(ws, nproc, sampler_name=None, dl_type=DataLoader):
+def _test_auto_dataloader(ws, nproc, batch_size, num_workers=1, sampler_name=None, dl_type=DataLoader):
 
     data = torch.rand(100, 3, 12, 12)
 
@@ -25,14 +25,22 @@ def _test_auto_dataloader(ws, nproc, sampler_name=None, dl_type=DataLoader):
 
     # Test auto_dataloader
     assert idist.get_world_size() == ws
-    dataloader = auto_dataloader(data, batch_size=10, num_workers=2, sampler=sampler, shuffle=sampler is None)
+    dataloader = auto_dataloader(
+        data, batch_size=batch_size, num_workers=num_workers, sampler=sampler, shuffle=sampler is None
+    )
 
     assert isinstance(dataloader, dl_type)
     if hasattr(dataloader, "_loader"):
         dataloader = dataloader._loader
+    if ws < batch_size:
+        assert dataloader.batch_size == batch_size // ws
+    else:
+        assert dataloader.batch_size == batch_size
+    if ws <= num_workers:
+        assert dataloader.num_workers == (num_workers + nproc - 1) // nproc
+    else:
+        assert dataloader.num_workers == num_workers
 
-    assert dataloader.batch_size == 10 // ws
-    assert dataloader.num_workers == (2 + nproc - 1) // nproc
     if ws < 2:
         sampler_type = RandomSampler if sampler is None else type(sampler)
         assert isinstance(dataloader.sampler, sampler_type)
@@ -69,8 +77,9 @@ def _test_auto_model_optimizer(ws, device):
 
 def test_auto_methods_no_dist():
 
-    _test_auto_dataloader(1, 1)
-    _test_auto_dataloader(1, 1, sampler_name="WeightedRandomSampler")
+    _test_auto_dataloader(1, 1, batch_size=1)
+    _test_auto_dataloader(1, 1, batch_size=10, num_workers=10)
+    _test_auto_dataloader(1, 1, batch_size=10, sampler_name="WeightedRandomSampler")
 
     _test_auto_model_optimizer(1, "cpu")
 
@@ -80,8 +89,9 @@ def test_auto_methods_no_dist():
 def test_auto_methods_gloo(distributed_context_single_node_gloo):
 
     ws = distributed_context_single_node_gloo["world_size"]
-    _test_auto_dataloader(ws=ws, nproc=ws)
-    _test_auto_dataloader(ws=ws, nproc=ws, sampler_name="WeightedRandomSampler")
+    _test_auto_dataloader(ws=ws, nproc=ws, batch_size=1)
+    _test_auto_dataloader(ws=ws, nproc=ws, batch_size=10, num_workers=10)
+    _test_auto_dataloader(ws=ws, nproc=ws, batch_size=10, sampler_name="WeightedRandomSampler")
 
     _test_auto_model_optimizer(ws, "cpu")
 
@@ -93,8 +103,9 @@ def test_auto_methods_nccl(distributed_context_single_node_nccl):
 
     ws = distributed_context_single_node_nccl["world_size"]
     lrank = distributed_context_single_node_nccl["local_rank"]
-    _test_auto_dataloader(ws=ws, nproc=ws)
-    _test_auto_dataloader(ws=ws, nproc=ws, sampler_name="WeightedRandomSampler")
+    _test_auto_dataloader(ws=ws, nproc=ws, batch_size=1)
+    _test_auto_dataloader(ws=ws, nproc=ws, batch_size=10, num_workers=10)
+    _test_auto_dataloader(ws=ws, nproc=ws, batch_size=1, sampler_name="WeightedRandomSampler")
 
     device = "cuda"
     _test_auto_model_optimizer(ws, device)
@@ -115,8 +126,9 @@ def _test_auto_methods_xla(index, ws):
         except ImportError:
             pass
 
-    _test_auto_dataloader(ws=ws, nproc=ws, dl_type=dl_type)
-    _test_auto_dataloader(ws=ws, nproc=ws, sampler_name="WeightedRandomSampler", dl_type=dl_type)
+    _test_auto_dataloader(ws=ws, nproc=ws, batch_size=1, dl_type=dl_type)
+    _test_auto_dataloader(ws=ws, nproc=ws, batch_size=10, num_workers=10, dl_type=dl_type)
+    _test_auto_dataloader(ws=ws, nproc=ws, batch_size=1, sampler_name="WeightedRandomSampler", dl_type=dl_type)
 
     device = "xla"
     _test_auto_model_optimizer(ws, device)