diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index a70ebdaba..ac79d3e0a 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -82,9 +82,10 @@ class Accelerator:
 
             Will default to :obj:`["torch"]` for PyTorch versions <=1.5.1 and :obj:`["generator"]` for PyTorch versions
             >= 1.6.
-        dispatch_batches (:obj:`bool`, `optional`, defaults to :obj:`False`):
+        dispatch_batches (:obj:`bool`, `optional`):
             If set to :obj:`True`, the dataloader prepared by the Accelerator is only iterated through on the main
-            process and then the batches are split and broadcast to each process.
+            process and then the batches are split and broadcast to each process. Will default to :obj:`True` for
+            :obj:`DataLoader` whose underlying dataset is an :obj:`IterableDataset`, :obj:`False` otherwise.
         kwargs_handlers (list of kwargs handlers, `optional`)
             A list of :obj:`KwargHandler` to customize how the objects related to distributed training or mixed
             precision are created. See :doc:`kwargs` for more information.
@@ -103,7 +104,7 @@ def __init__(
         cpu: bool = False,
         deepspeed_plugin: DeepSpeedPlugin = None,
         rng_types: Optional[List[Union[str, RNGType]]] = None,
-        dispatch_batches: bool = False,
+        dispatch_batches: Optional[bool] = None,
         kwargs_handlers: Optional[List[KwargsHandler]] = None,
     ):
         if deepspeed_plugin is None:  # init from env variables
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index 1d464c55f..03e38baf8 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -398,7 +398,7 @@ def prepare_data_loader(
     split_batches: bool = False,
     put_on_device: bool = False,
     rng_types: Optional[List[Union[str, RNGType]]] = None,
-    dispatch_batches: bool = False,
+    dispatch_batches: Optional[bool] = None,
 ) -> DataLoader:
     """
     Wraps a PyTorch :obj:`DataLoader` to generate batches for one of the processes only.
@@ -441,9 +441,10 @@ def prepare_data_loader(
             - :obj:`"generator"`: the :obj:`torch.Generator` of the sampler (or batch sampler if there is no sampler in
               your dataloader) or of the iterable dataset (if it exists) if the underlying dataset is of that type.
 
-        dispatch_batches (:obj:`bool`, `optional`, defaults to :obj:`False`):
+        dispatch_batches (:obj:`bool`, `optional`):
             If set to :obj:`True`, the datalaoder prepared is only iterated through on the main process and then the
-            batches are split and broadcast to each process.
+            batches are split and broadcast to each process. Will default to :obj:`True` when the underlying dataset is
+            an :obj:`IterableDataset`, :obj:`False` otherwise.
 
     Returns:
         :obj:`torch.utils.data.dataloader.DataLoader`: A new data loader that will yield the portion of the batches
@@ -452,6 +453,9 @@ def prepare_data_loader(
 
         This does not support :obj:`BatchSampler` with varying batch size yet.
     """
+    if dispatch_batches is None:
+        dispatch_batches = False if not put_on_device else isinstance(dataloader.dataset, IterableDataset)
+
     if dispatch_batches and not put_on_device:
         raise ValueError("Using `dispatch_batches=True` requires `put_on_device=True`.")
     # Grab defaults from AcceleratorState


Review comments: body: Clean!
diff_hunk: @@ -452,6 +453,9 @@ def prepare_data_loader(
 
         This does not support :obj:`BatchSampler` with varying batch size yet.
     """
+    if dispatch_batches is None:
+        dispatch_batches = False if not put_on_device else isinstance(new_dataset, IterableDataset)