diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index a35f7e268..a70ebdaba 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -82,6 +82,9 @@ class Accelerator:
 
             Will default to :obj:`["torch"]` for PyTorch versions <=1.5.1 and :obj:`["generator"]` for PyTorch versions
             >= 1.6.
+        dispatch_batches (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            If set to :obj:`True`, the dataloader prepared by the Accelerator is only iterated through on the main
+            process and then the batches are split and broadcast to each process.
         kwargs_handlers (list of kwargs handlers, `optional`)
             A list of :obj:`KwargHandler` to customize how the objects related to distributed training or mixed
             precision are created. See :doc:`kwargs` for more information.
@@ -100,6 +103,7 @@ def __init__(
         cpu: bool = False,
         deepspeed_plugin: DeepSpeedPlugin = None,
         rng_types: Optional[List[Union[str, RNGType]]] = None,
+        dispatch_batches: bool = False,
         kwargs_handlers: Optional[List[KwargsHandler]] = None,
     ):
         if deepspeed_plugin is None:  # init from env variables
@@ -113,6 +117,7 @@ def __init__(
 
         self.device_placement = device_placement
         self.split_batches = split_batches
+        self.dispatch_batches = dispatch_batches
 
         # Kwargs handlers
         self.ddp_handler = None
@@ -384,6 +389,7 @@ def prepare_data_loader(self, data_loader):
             split_batches=self.split_batches,
             put_on_device=self.device_placement,
             rng_types=self.rng_types.copy(),
+            dispatch_batches=self.dispatch_batches,
         )
 
     def prepare_optimizer(self, optimizer):
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index 96648ca65..fd10140b6 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -20,7 +20,18 @@
 from packaging import version
 
 from .state import AcceleratorState, DistributedType, is_tpu_available
-from .utils import RNGType, send_to_device, synchronize_rng_states
+from .utils import (
+    RNGType,
+    broadcast,
+    broadcast_object_list,
+    concatenate,
+    find_batch_size,
+    get_data_structure,
+    initialize_tensors,
+    send_to_device,
+    slice_tensors,
+    synchronize_rng_states,
+)
 
 
 if is_tpu_available():
@@ -294,6 +305,90 @@ def __iter__(self):
             yield batch if self.device is None else send_to_device(batch, self.device)
 
 
+class DataLoaderDispatcher(DataLoader):
+    """
+    Subclass of a PyTorch :obj:`DataLoader` that will iterate and preprocess on process 0 only, then dispatch on each
+    process their part of the batch.
+
+    Args:
+        split_batches (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether the resulting :obj:`DataLoader` should split the batches of the original data loader across devices
+            or yield full batches (in which case it will yield batches starting at the :obj:`process_index`-th and
+            advancing of :obj:`num_processes` batches at each iteration).
+
+            Another way to see this is that the observed batch size will be the same as the initial :obj:`dataloader`
+            if this option is set to :obj:`True`, the batch size of the initial :obj:`dataloader` multiplied by
+            :obj:`num_processes` otherwise.
+
+            Setting this option to :obj:`True` requires that the batch size of the :obj:`dataloader` is a round
+            multiple of :obj:`batch_size`.
+    """
+
+    def __init__(self, dataset, split_batches: bool = False, **kwargs):
+        super().__init__(dataset, **kwargs)
+        self.split_batches = split_batches
+
+    def __iter__(self):
+        state = AcceleratorState()
+        if state.process_index == 0:
+            # We only iterate through the DataLoader on process 0.
+            main_iterator = super().__iter__()
+        stop_iteration = False
+        while not stop_iteration:
+            # On process 0, we gather the batch to dispatch.
+            if state.process_index == 0:
+                try:
+                    if self.split_batches:
+                        # One batch of the main iterator is dispatched and split.
+                        batch = next(main_iterator)
+                    else:
+                        # num_processes batches of the main iterator are concatenated then dispatched and split.
+                        # We add the batches one by one so we have the remainder available when drop_last=False.
+                        batches = []
+                        for _ in range(state.num_processes):
+                            batches.append(next(main_iterator))
+                        batch = concatenate(batches, dim=0)
+                    # In both cases, we need to get the structure of the batch that we will broadcast on other
+                    # processes to initialize the tensors with the right shape.
+                    # data_structure, stop_iteration
+                    batch_info = [get_data_structure(batch), False]
+                except StopIteration:
+                    batch_info = [None, True]
+            else:
+                batch_info = [None, stop_iteration]
+
+            # This is inplace, so after this instruction, every process has the same `batch_info` as process 0.
+            broadcast_object_list(batch_info)
+            stop_iteration = batch_info[1]
+            if stop_iteration:
+                # If drop_last is False and split_batches is False, we may have a remainder to take care of.
+                if not self.split_batches and not self.drop_last:
+                    if state.process_index == 0 and len(batches) > 0:
+                        batch = concatenate(batches, dim=0)
+                        batch_info = [get_data_structure(batch), False]
+                    else:
+                        batch_info = [None, True]
+                    broadcast_object_list(batch_info)
+                    if batch_info[1]:
+                        continue
+                else:
+                    continue
+
+            if state.process_index != 0:
+                # Initialize tensors on other processes than process 0.
+                batch = initialize_tensors(batch_info[0])
+            batch = send_to_device(batch, state.device)
+            # Broadcast the batch before splitting it.
+            batch = broadcast(batch, from_process=0)
+
+            batch_size = find_batch_size(batch) // state.num_processes
+            data_slice = slice(state.process_index * batch_size, (state.process_index + 1) * batch_size)
+
+            if state.distributed_type == DistributedType.TPU:
+                xm.mark_step()
+            yield slice_tensors(batch, data_slice)
+
+
 def prepare_data_loader(
     dataloader: DataLoader,
     device: Optional[torch.device] = None,
@@ -302,6 +397,7 @@ def prepare_data_loader(
     split_batches: bool = False,
     put_on_device: bool = False,
     rng_types: Optional[List[Union[str, RNGType]]] = None,
+    dispatch_batches: bool = False,
 ) -> DataLoader:
     """
     Wraps a PyTorch :obj:`DataLoader` to generate batches for one of the processes only.
@@ -344,6 +440,10 @@ def prepare_data_loader(
             - :obj:`"generator"`: the :obj:`torch.Generator` of the sampler (or batch sampler if there is no sampler in
               your dataloader) or of the iterable dataset (if it exists) if the underlying dataset is of that type.
 
+        dispatch_batches (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            If set to :obj:`True`, the datalaoder prepared is only iterated through on the main process and then the
+            batches are split and broadcast to each process.
+
     Returns:
         :obj:`torch.utils.data.dataloader.DataLoader`: A new data loader that will yield the portion of the batches
 
@@ -351,6 +451,8 @@ def prepare_data_loader(
 
         This does not support :obj:`BatchSampler` with varying batch size yet.
     """
+    if dispatch_batches and not put_on_device:
+        raise ValueError("Using `dispatch_batches=True` requires `put_on_device=True`.")
     # Grab defaults from AcceleratorState
     state = AcceleratorState()
     if num_processes is None:
@@ -370,7 +472,7 @@ def prepare_data_loader(
     new_batch_sampler = dataloader.batch_sampler if not isinstance(new_dataset, IterableDataset) else None
     generator = getattr(dataloader, "generator", None)
     # No change if no multiprocess
-    if num_processes != 1:
+    if num_processes != 1 and not dispatch_batches:
         if isinstance(new_dataset, IterableDataset):
             if getattr(dataloader.dataset, "generator", None) is not None:
                 generator = dataloader.dataset.generator
@@ -421,6 +523,11 @@ def prepare_data_loader(
     if new_batch_sampler is None:
         kwargs["batch_size"] = dataloader.batch_size // num_processes if split_batches else dataloader.batch_size
 
+    if dispatch_batches:
+        return DataLoaderDispatcher(
+            new_dataset, split_batches=split_batches, batch_sampler=new_batch_sampler, **kwargs
+        )
+
     return DataLoaderShard(
         new_dataset,
         device=device if put_on_device else None,
diff --git a/src/accelerate/test_utils/test_script.py b/src/accelerate/test_utils/test_script.py
index 2e5095533..41efcf61c 100644
--- a/src/accelerate/test_utils/test_script.py
+++ b/src/accelerate/test_utils/test_script.py
@@ -36,14 +36,14 @@ def init_state_check():
 def rng_sync_check():
     state = AcceleratorState()
     synchronize_rng_states(["torch"])
-    assert are_the_same_tensors(torch.get_rng_state())
+    assert are_the_same_tensors(torch.get_rng_state()), "RNG states improperly synchronized on CPU."
     if state.distributed_type == DistributedType.MULTI_GPU:
         synchronize_rng_states(["cuda"])
-        assert are_the_same_tensors(torch.cuda.get_rng_state())
+        assert are_the_same_tensors(torch.cuda.get_rng_state()), "RNG states improperly synchronized on GPU."
     if version.parse(torch.__version__) >= version.parse("1.6.0"):
         generator = torch.Generator()
         synchronize_rng_states(["generator"], generator=generator)
-        assert are_the_same_tensors(generator.get_state())
+        assert are_the_same_tensors(generator.get_state()), "RNG states improperly synchronized in generator."
 
     if state.local_process_index == 0:
         print("All rng are properly synched.")
@@ -59,7 +59,7 @@ def dl_preparation_check():
     for batch in dl:
         result.append(gather(batch))
     result = torch.cat(result)
-    assert torch.equal(result.cpu(), torch.arange(0, length).long())
+    assert torch.equal(result.cpu(), torch.arange(0, length).long()), "Wrong non-shuffled dataloader result."
 
     dl = DataLoader(range(length), batch_size=8)
     dl = prepare_data_loader(
@@ -74,7 +74,7 @@ def dl_preparation_check():
     for batch in dl:
         result.append(gather(batch))
     result = torch.cat(result)
-    assert torch.equal(result.cpu(), torch.arange(0, length).long())
+    assert torch.equal(result.cpu(), torch.arange(0, length).long()), "Wrong non-shuffled dataloader result."
 
     if state.process_index == 0:
         print("Non-shuffled dataloader passing.")
@@ -86,7 +86,7 @@ def dl_preparation_check():
         result.append(gather(batch))
     result = torch.cat(result).tolist()
     result.sort()
-    assert result == list(range(length))
+    assert result == list(range(length)), "Wrong shuffled dataloader result."
 
     dl = DataLoader(range(length), batch_size=8, shuffle=True)
     dl = prepare_data_loader(
@@ -102,12 +102,77 @@ def dl_preparation_check():
         result.append(gather(batch))
     result = torch.cat(result).tolist()
     result.sort()
-    assert result == list(range(length))
+    assert result == list(range(length)), "Wrong shuffled dataloader result."
 
     if state.local_process_index == 0:
         print("Shuffled dataloader passing.")
 
 
+def central_dl_preparation_check():
+    state = AcceleratorState()
+    length = 32 * state.num_processes
+
+    dl = DataLoader(range(length), batch_size=8)
+    dl = prepare_data_loader(
+        dl, state.device, state.num_processes, state.process_index, put_on_device=True, dispatch_batches=True
+    )
+    result = []
+    for batch in dl:
+        result.append(gather(batch))
+    result = torch.cat(result)
+    assert torch.equal(result.cpu(), torch.arange(0, length).long()), "Wrong non-shuffled dataloader result."
+
+    dl = DataLoader(range(length), batch_size=8)
+    dl = prepare_data_loader(
+        dl,
+        state.device,
+        state.num_processes,
+        state.process_index,
+        put_on_device=True,
+        split_batches=True,
+        dispatch_batches=True,
+    )
+    result = []
+    for batch in dl:
+        result.append(gather(batch))
+    result = torch.cat(result)
+    assert torch.equal(result.cpu(), torch.arange(0, length).long()), "Wrong non-shuffled dataloader result."
+
+    if state.process_index == 0:
+        print("Non-shuffled central dataloader passing.")
+
+    dl = DataLoader(range(length), batch_size=8, shuffle=True)
+    dl = prepare_data_loader(
+        dl, state.device, state.num_processes, state.process_index, put_on_device=True, dispatch_batches=True
+    )
+    result = []
+    for batch in dl:
+        result.append(gather(batch))
+    result = torch.cat(result).tolist()
+    result.sort()
+    assert result == list(range(length)), "Wrong shuffled dataloader result."
+
+    dl = DataLoader(range(length), batch_size=8, shuffle=True)
+    dl = prepare_data_loader(
+        dl,
+        state.device,
+        state.num_processes,
+        state.process_index,
+        put_on_device=True,
+        split_batches=True,
+        dispatch_batches=True,
+    )
+    result = []
+    for batch in dl:
+        result.append(gather(batch))
+    result = torch.cat(result).tolist()
+    result.sort()
+    assert result == list(range(length)), "Wrong shuffled dataloader result."
+
+    if state.local_process_index == 0:
+        print("Shuffled central dataloader passing.")
+
+
 def mock_training(length, batch_size, generator):
     set_seed(42)
     generator.manual_seed(42)
@@ -132,8 +197,8 @@ def training_check():
     length = batch_size * 4 * state.num_processes
 
     train_set, old_model = mock_training(length, batch_size * state.num_processes, generator)
-    assert are_the_same_tensors(old_model.a)
-    assert are_the_same_tensors(old_model.b)
+    assert are_the_same_tensors(old_model.a), "Did not obtain the same model on both processes."
+    assert are_the_same_tensors(old_model.b), "Did not obtain the same model on both processes."
 
     accelerator = Accelerator()
     train_dl = DataLoader(train_set, batch_size=batch_size, shuffle=True, generator=generator)
@@ -152,8 +217,8 @@ def training_check():
             optimizer.step()
 
     model = accelerator.unwrap_model(model).cpu()
-    assert torch.allclose(old_model.a, model.a)
-    assert torch.allclose(old_model.b, model.b)
+    assert torch.allclose(old_model.a, model.a), "Did not obtain the same model on CPU or distributed training."
+    assert torch.allclose(old_model.b, model.b), "Did not obtain the same model on CPU or distributed training."
 
     accelerator.print("Training yielded the same results on one CPU or distributed setup with no batch split.")
 
@@ -174,8 +239,8 @@ def training_check():
             optimizer.step()
 
     model = accelerator.unwrap_model(model).cpu()
-    assert torch.allclose(old_model.a, model.a)
-    assert torch.allclose(old_model.b, model.b)
+    assert torch.allclose(old_model.a, model.a), "Did not obtain the same model on CPU or distributed training."
+    assert torch.allclose(old_model.b, model.b), "Did not obtain the same model on CPU or distributed training."
 
     accelerator.print("Training yielded the same results on one CPU or distributes setup with batch split.")
 
@@ -197,8 +262,8 @@ def training_check():
             optimizer.step()
 
     model = accelerator.unwrap_model(model).cpu()
-    assert torch.allclose(old_model.a, model.a)
-    assert torch.allclose(old_model.b, model.b)
+    assert torch.allclose(old_model.a, model.a), "Did not obtain the same model on CPU or distributed training."
+    assert torch.allclose(old_model.b, model.b), "Did not obtain the same model on CPU or distributed training."
 
 
 def main():
@@ -215,6 +280,7 @@ def main():
     if state.local_process_index == 0:
         print("\n**DataLoader integration test**")
     dl_preparation_check()
+    central_dl_preparation_check()
 
     if state.local_process_index == 0:
         print("\n**Training integration test**")
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 026ab9501..8df05578a 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -48,6 +48,12 @@ class RNGType(Enum):
     GENERATOR = "generator"
 
 
+@dataclass
+class TensorInformation:
+    shape: torch.Size
+    dtype: torch.dtype
+
+
 def set_seed(seed: int):
     """
     Helper function for reproducible behavior to set the seed in ``random``, ``numpy``, ``torch``.
@@ -115,6 +121,66 @@ def honor_type(obj, generator):
     return type(obj)(generator)
 
 
+def is_torch_tensor(tensor):
+    return isinstance(tensor, torch.Tensor)
+
+
+def is_tensor_information(tensor_info):
+    return isinstance(tensor_info, TensorInformation)
+
+
+def recursively_apply(func, data, *args, test_type=is_torch_tensor, error_on_other_type=False, **kwargs):
+    """
+    Recursively apply a function on a data structure that is a nested list/tuple/dictionary of a given base type.
+
+    Args:
+        func (:obj:`callable`):
+            The function to recursively apply.
+        data (nested list/tuple/dictionary of :obj:`main_type`):
+            The data on which to apply :obj:`func`
+        *args:
+            Positional arguments that will be passed to :obj:`func` when applied on the unpacked data.
+        main_type (:obj:`type`, `optional`, defaults to :obj:`torch.Tensor`):
+            The base type of the objects to which apply :obj:`func`.
+        error_on_other_type (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether to return an error or not if after unpacking :obj:`data`, we get on an object that is not of type
+            :obj:`main_type`. If :obj:`False`, the function will leave objects of types different than :obj:`main_type`
+            unchanged.
+        **kwargs:
+            Keyword arguments that will be passed to :obj:`func` when applied on the unpacked data.
+
+    Returns:
+        The same data structure as :obj:`data` with :obj:`func` applied to every object of type :obj:`main_type`.
+    """
+    if isinstance(data, (tuple, list)):
+        return honor_type(
+            data,
+            (
+                recursively_apply(
+                    func, o, *args, test_type=test_type, error_on_other_type=error_on_other_type, **kwargs
+                )
+                for o in data
+            ),
+        )
+    elif isinstance(data, dict):
+        return type(data)(
+            **{
+                k: recursively_apply(
+                    func, v, *args, test_type=test_type, error_on_other_type=error_on_other_type, **kwargs
+                )
+                for k, v in data.items()
+            }
+        )
+    elif test_type(data):
+        return func(data, *args, **kwargs)
+    elif error_on_other_type:
+        raise TypeError(
+            f"Can't apply {func.__name__} on object of type {type(data)}, only of nested list/tuple/dicts of objects "
+            f"that satisfy {test_type.__name__}."
+        )
+    return data
+
+
 def send_to_device(tensor, device):
     """
     Recursively sends the elements in a nested list/tuple/dictionary of tensors to a given device.
@@ -128,18 +194,52 @@ def send_to_device(tensor, device):
     Returns:
         The same data structure as :obj:`tensor` with all tensors sent to the proper device.
     """
-    if isinstance(tensor, (list, tuple)):
-        return honor_type(tensor, (send_to_device(t, device) for t in tensor))
-    elif isinstance(tensor, dict):
-        return type(tensor)({k: send_to_device(v, device) for k, v in tensor.items()})
-    elif not hasattr(tensor, "to"):
-        return tensor
-    return tensor.to(device)
+
+    def _send_to_device(t, device):
+        return t.to(device)
+
+    def _has_to_method(t):
+        return hasattr(t, "to")
+
+    return recursively_apply(_send_to_device, tensor, device, test_type=_has_to_method, error_on_other_type=True)
+
+
+def get_data_structure(data):
+    """
+    Recursively gathers the information needed to rebuild a nested list/tuple/dictionary of tensors.
+
+    Args:
+        data (nested list/tuple/dictionary of :obj:`torch.Tensor`):
+            The data to send to analyze.
+
+    Returns:
+        The same data structure as :obj:`data` with :class:`~accelerate.utils.TensorInformation` instead of tensors.
+    """
+
+    def _get_data_structure(tensor):
+        return TensorInformation(shape=tensor.shape, dtype=tensor.dtype)
+
+    return recursively_apply(_get_data_structure, data)
+
+
+def initialize_tensors(data_structure):
+    """
+    Recursively initializes tensors from a nested list/tuple/dictionary of
+    :class:`~accelerate.utils.TensorInformation`.
+
+    Returns:
+        The same data structure as :obj:`data` with tensors instead of :class:`~accelerate.utils.TensorInformation`.
+    """
+
+    def _initialize_tensor(tensor_info):
+        return torch.empty(*tensor_info.shape, dtype=tensor_info.dtype)
+
+    return recursively_apply(_initialize_tensor, data_structure, test_type=is_tensor_information)
 
 
 def convert_to_fp32(tensor):
     """
-    Recursively converts the lements nested list/tuple/dictionary of tensors in FP16 precision to FP32.
+    Recursively converts the elements nested list/tuple/dictionary of tensors in FP16 precision to FP32.
 
     Args:
         tensor (nested list/tuple/dictionary of :obj:`torch.Tensor`):
@@ -148,13 +248,14 @@ def convert_to_fp32(tensor):
     Returns:
         The same data structure as :obj:`tensor` with all tensors that were in FP16 precision converted to FP32.
     """
-    if isinstance(tensor, (list, tuple)):
-        return honor_type(tensor, (convert_to_fp32(t) for t in tensor))
-    elif isinstance(tensor, dict):
-        return type(tensor)(**{k: convert_to_fp32(v) for k, v in tensor.items()})
-    elif not hasattr(tensor, "dtype") or tensor.dtype != torch.float16:
-        return tensor
-    return tensor.float()
+
+    def _convert_to_fp32(tensor):
+        return tensor.float()
+
+    def _is_fp16_tensor(tensor):
+        return hasattr(tensor, "dtype") and tensor.dtype == torch.float16
+
+    return recursively_apply(_is_fp16_tensor, tensor, test_type=_is_fp16_tensor)
 
 
 def convert_outputs_to_fp32(model_forward):
@@ -196,7 +297,7 @@ def extract_model_from_parallel(model):
     return model
 
 
-def _tpu_gather(tensor, name="tensor"):
+def _tpu_gather(tensor, name="gather tensor"):
     if isinstance(tensor, (list, tuple)):
         return honor_type(tensor, (_tpu_gather(t, name=f"{name}_{i}") for i, t in enumerate(tensor)))
     elif isinstance(tensor, dict):
@@ -209,17 +310,14 @@ def _tpu_gather(tensor, name="tensor"):
 
 
 def _gpu_gather(tensor):
-    if isinstance(tensor, (list, tuple)):
-        return honor_type(tensor, (_gpu_gather(t) for t in tensor))
-    elif isinstance(tensor, dict):
-        return type(tensor)({k: _gpu_gather(v) for k, v in tensor.items()})
-    elif not isinstance(tensor, torch.Tensor):
-        raise TypeError(f"Can't gather the values of type {type(tensor)}, only of nested list/tuple/dicts of tensors.")
-    if tensor.ndim == 0:
-        tensor = tensor.clone()[None]
-    output_tensors = [tensor.clone() for _ in range(torch.distributed.get_world_size())]
-    torch.distributed.all_gather(output_tensors, tensor)
-    return torch.cat(output_tensors, dim=0)
+    def _gpu_gather_one(tensor):
+        if tensor.ndim == 0:
+            tensor = tensor.clone()[None]
+        output_tensors = [tensor.clone() for _ in range(torch.distributed.get_world_size())]
+        torch.distributed.all_gather(output_tensors, tensor)
+        return torch.cat(output_tensors, dim=0)
+
+    return recursively_apply(_gpu_gather_one, tensor, error_on_other_type=True)
 
 
 _cpu_gather = _gpu_gather
@@ -246,6 +344,130 @@ def gather(tensor):
         return tensor
 
 
+def _gpu_broadcast(data, src=0):
+    def _gpu_broadcast_one(tensor, src=0):
+        torch.distributed.broadcast(tensor, src=src)
+        return tensor
+
+    return recursively_apply(_gpu_broadcast_one, data, error_on_other_type=True, src=src)
+
+
+def _tpu_broadcast(tensor, src=0, name="broadcast tensor"):
+    if isinstance(tensor, (list, tuple)):
+        return honor_type(tensor, (_tpu_broadcast(t, name=f"{name}_{i}") for i, t in enumerate(tensor)))
+    elif isinstance(tensor, dict):
+        return type(tensor)({k: _tpu_broadcast(v, name=f"{name}_{k}") for k, v in tensor.items()})
+    return xm.mesh_reduce(name, tensor, lambda x: x[src])
+
+
+def broadcast(tensor, from_process: int = 0):
+    """
+    Recursively broadcast tensor in a nested list/tuple/dictionary of tensors to all devices.
+
+    Args:
+        tensor (nested list/tuple/dictionary of :obj:`torch.Tensor`):
+            The data to gather.
+        from_process (:obj:`int`, `optional`, defaults to 0):
+            The process from which to send the data
+
+    Returns:
+        The same data structure as :obj:`tensor` with all tensors broadcasted to the proper device.
+    """
+    if AcceleratorState().distributed_type == DistributedType.TPU:
+        return _tpu_broadcast(tensor, src=from_process, name="accelerate.utils.broadcast")
+    elif AcceleratorState().distributed_type == DistributedType.MULTI_GPU:
+        return _gpu_broadcast(tensor, src=from_process)
+    elif AcceleratorState().distributed_type == DistributedType.MULTI_CPU:
+        return _gpu_broadcast(tensor, src=from_process)
+    else:
+        return tensor
+
+
+def broadcast_object_list(object_list, from_process: int = 0):
+    """
+    Broadcast a list of picklable objects form one process to the others.
+
+    Args:
+        object_list (list of picklable objects):
+            The list of objects to broadcast. This list will be modified inplace.
+        from_process (:obj:`int`, `optional`, defaults to 0):
+            The process from which to send the data.
+
+    Returns:
+        The same list containing the objects from process 0.
+    """
+    if AcceleratorState().distributed_type == DistributedType.TPU:
+        for i, obj in enumerate(object_list):
+            object_list[i] = xm.mesh_reduce("accelerate.utils.broadcast_object_list", obj, lambda x: x[from_process])
+    elif AcceleratorState().distributed_type == DistributedType.MULTI_GPU:
+        torch.distributed.broadcast_object_list(object_list, src=from_process)
+    elif AcceleratorState().distributed_type == DistributedType.MULTI_CPU:
+        torch.distributed.broadcast_object_list(object_list, src=from_process)
+    return object_list
+
+
+def slice_tensors(data, tensor_slice):
+    """
+    Recursively takes a slice in a nested list/tuple/dictionary of tensors.
+
+    Args:
+        data (nested list/tuple/dictionary of :obj:`torch.Tensor`):
+            The data to slice.
+        tensor_slice (:obj:`slice`):
+            The slice to take.
+
+    Returns:
+        The same data structure as :obj:`data` with all the tensors slices.
+    """
+
+    def _slice_tensor(tensor, tensor_slice):
+        return tensor[tensor_slice]
+
+    return recursively_apply(_slice_tensor, data, tensor_slice)
+
+
+def find_batch_size(data):
+    """
+    Recursively finds the batch size in a nested list/tuple/dictionary of lists of tensors.
+
+    Args:
+        data (nested list/tuple/dictionary of :obj:`torch.Tensor`): The data from which to find the batch size.
+
+    Returns:
+        :obj:`int`: The batch size.
+    """
+    if isinstance(data, (tuple, list)):
+        return find_batch_size(data[0])
+    elif isinstance(data, dict):
+        for k in data.keys():
+            return find_batch_size(data[k])
+    elif not isinstance(data, torch.Tensor):
+        raise TypeError(f"Can only find the batch size of tensors but got {type(data)}.")
+    return data.shape[0]
+
+
+def concatenate(data, dim=0):
+    """
+    Recursively concatenate the tensors in a nested list/tuple/dictionary of lists of tensors with the same shape.
+
+    Args:
+        data (nested list/tuple/dictionary of lists of tensors :obj:`torch.Tensor`):
+            The data to concatenate.
+        dim (:obj:`int`, `optional`, defaults to 0):
+            The dimension on which to concatenate.
+
+    Returns:
+        The same data structure as :obj:`data` with all the tensors concatenated.
+    """
+    if isinstance(data[0], (tuple, list)):
+        return honor_type(data[0], (concatenate([d[i] for d in data], dim=dim) for i in range(len(data[0]))))
+    elif isinstance(data[0], dict):
+        return type(data[0])(**{k: concatenate([d[k] for d in data], dim=dim) for k in data[0].keys()})
+    elif not isinstance(data[0], torch.Tensor):
+        raise TypeError(f"Can only concatenate tensors but got {type(data[0])}")
+    return torch.cat(data, dim=dim)
+
+
 def pad_across_processes(tensor, dim=0, pad_index=0, pad_first=False):
     """
     Recursively pad the tensors in a nested list/tuple/dictionary of tensors from all devices to the same size so they
@@ -261,36 +483,35 @@ def pad_across_processes(tensor, dim=0, pad_index=0, pad_first=False):
         pad_first (:obj:`bool`, `optional`, defaults to :obj:`False`):
             Whether to pad at the beginning or the end.
     """
-    if isinstance(tensor, (list, tuple)):
-        return honor_type(tensor, (pad_across_processes(t, dim=dim, pad_index=pad_index) for t in tensor))
-    elif isinstance(tensor, dict):
-        return type(tensor)({k: pad_across_processes(v, dim=dim, pad_index=pad_index) for k, v in tensor.items()})
-    elif not isinstance(tensor, torch.Tensor):
-        raise TypeError(f"Can't pad the values of type {type(tensor)}, only of nested list/tuple/dicts of tensors.")
 
-    if dim >= len(tensor.shape):
-        return tensor
-
-    # Gather all sizes
-    size = torch.tensor(tensor.shape, device=tensor.device)[None]
-    sizes = gather(size).cpu()
-    # Then pad to the maximum size
-    max_size = max(s[dim] for s in sizes)
-    if max_size == tensor.shape[dim]:
-        return tensor
-
-    old_size = tensor.shape
-    new_size = list(old_size)
-    new_size[dim] = max_size
-    new_tensor = tensor.new_zeros(tuple(new_size)) + pad_index
-    if pad_first:
-        indices = tuple(
-            slice(max_size - old_size[dim], max_size) if i == dim else slice(None) for i in range(len(new_size))
-        )
-    else:
-        indices = tuple(slice(0, old_size[dim]) if i == dim else slice(None) for i in range(len(new_size)))
-    new_tensor[indices] = tensor
-    return new_tensor
+    def _pad_across_processes(tensor, dim=0, pad_index=0, pad_first=False):
+        if dim >= len(tensor.shape):
+            return tensor
+
+        # Gather all sizes
+        size = torch.tensor(tensor.shape, device=tensor.device)[None]
+        sizes = gather(size).cpu()
+        # Then pad to the maximum size
+        max_size = max(s[dim] for s in sizes)
+        if max_size == tensor.shape[dim]:
+            return tensor
+
+        old_size = tensor.shape
+        new_size = list(old_size)
+        new_size[dim] = max_size
+        new_tensor = tensor.new_zeros(tuple(new_size)) + pad_index
+        if pad_first:
+            indices = tuple(
+                slice(max_size - old_size[dim], max_size) if i == dim else slice(None) for i in range(len(new_size))
+            )
+        else:
+            indices = tuple(slice(0, old_size[dim]) if i == dim else slice(None) for i in range(len(new_size)))
+        new_tensor[indices] = tensor
+        return new_tensor
+
+    return recursively_apply(
+        _pad_across_processes, tensor, error_on_other_type=True, dim=dim, pad_index=pad_index, pad_first=pad_first
+    )
 
 
 def wait_for_everyone():


Review comments: body: ```suggestion
            If set to :obj:`True`, the dataloader prepared by the Accelerator is only iterated through on the main
```
diff_hunk: @@ -82,6 +82,9 @@ class Accelerator:
 
             Will default to :obj:`["torch"]` for PyTorch versions <=1.5.1 and :obj:`["generator"]` for PyTorch versions
             >= 1.6.
+        central_dataloader (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            If set to :obj:`True`, the datalaoder prepared by the Accelerator is only iterated through on the main
body: I don't like `central` as I don't really understand what it means from a quick read, but I spent 10 minutes trying to find a better name without success, so fine to leave it like it is :)
diff_hunk: @@ -82,6 +82,9 @@ class Accelerator:
 
             Will default to :obj:`["torch"]` for PyTorch versions <=1.5.1 and :obj:`["generator"]` for PyTorch versions
             >= 1.6.
+        central_dataloader (:obj:`bool`, `optional`, defaults to :obj:`False`):
body: What is this `my_batch_size` used for?
diff_hunk: @@ -421,6 +524,13 @@ def prepare_data_loader(
     if new_batch_sampler is None:
         kwargs["batch_size"] = dataloader.batch_size // num_processes if split_batches else dataloader.batch_size
 
+    if central_dataloader:
+        data_loader = DataLoaderDispatcher(
+            new_dataset, split_batches=split_batches, batch_sampler=new_batch_sampler, **kwargs
+        )
+        data_loader.my_batch_size = dataloader.batch_size
body: Please add an error message if it fails :)
diff_hunk: @@ -108,6 +108,71 @@ def dl_preparation_check():
         print("Shuffled dataloader passing.")
 
 
+def central_dl_preparation_check():
+    state = AcceleratorState()
+    length = 32 * state.num_processes
+
+    dl = DataLoader(range(length), batch_size=8)
+    dl = prepare_data_loader(
+        dl, state.device, state.num_processes, state.process_index, put_on_device=True, central_dataloader=True
+    )
+    result = []
+    for batch in dl:
+        result.append(gather(batch))
+    result = torch.cat(result)
+    assert torch.equal(result.cpu(), torch.arange(0, length).long())
body: Oh actually this applies to all `assert` in this file!
diff_hunk: @@ -108,6 +108,71 @@ def dl_preparation_check():
         print("Shuffled dataloader passing.")
 
 
+def central_dl_preparation_check():
+    state = AcceleratorState()
+    length = 32 * state.num_processes
+
+    dl = DataLoader(range(length), batch_size=8)
+    dl = prepare_data_loader(
+        dl, state.device, state.num_processes, state.process_index, put_on_device=True, central_dataloader=True
+    )
+    result = []
+    for batch in dl:
+        result.append(gather(batch))
+    result = torch.cat(result)
+    assert torch.equal(result.cpu(), torch.arange(0, length).long())
body: Woopsie, this is some leftover to clean up! Thanks for flagging!
diff_hunk: @@ -421,6 +524,13 @@ def prepare_data_loader(
     if new_batch_sampler is None:
         kwargs["batch_size"] = dataloader.batch_size // num_processes if split_batches else dataloader.batch_size
 
+    if central_dataloader:
+        data_loader = DataLoaderDispatcher(
+            new_dataset, split_batches=split_batches, batch_sampler=new_batch_sampler, **kwargs
+        )
+        data_loader.my_batch_size = dataloader.batch_size
body: This is just a test script ;-)
diff_hunk: @@ -108,6 +108,71 @@ def dl_preparation_check():
         print("Shuffled dataloader passing.")
 
 
+def central_dl_preparation_check():
+    state = AcceleratorState()
+    length = 32 * state.num_processes
+
+    dl = DataLoader(range(length), batch_size=8)
+    dl = prepare_data_loader(
+        dl, state.device, state.num_processes, state.process_index, put_on_device=True, central_dataloader=True
+    )
+    result = []
+    for batch in dl:
+        result.append(gather(batch))
+    result = torch.cat(result)
+    assert torch.equal(result.cpu(), torch.arange(0, length).long())
body: Let's go for `dispath_batches` to look like `split_batches`
diff_hunk: @@ -82,6 +82,9 @@ class Accelerator:
 
             Will default to :obj:`["torch"]` for PyTorch versions <=1.5.1 and :obj:`["generator"]` for PyTorch versions
             >= 1.6.
+        central_dataloader (:obj:`bool`, `optional`, defaults to :obj:`False`):