diff --git a/docs/source/quicktour.rst b/docs/source/quicktour.rst
index 1e3ac4e89..31c94ef81 100644
--- a/docs/source/quicktour.rst
+++ b/docs/source/quicktour.rst
@@ -130,6 +130,13 @@ do with the :meth:`~accelerate.Accelerator.gather` method.
     Any instruction using your training dataloader length (for instance if you need the number of total training steps
     to create a learning rate scheduler) should go after the call to :meth:`~accelerate.Accelerator.prepare`.
 
+.. Warning::
+
+    The :meth:`~accelerate.Accelerator.gather` method requires the tensors to be all the same size on each process. If
+    you have tensors of different sizes on each process (for instance when dynamically padding to the maximum length in
+    a batch), you should use the :meth:`~accelerate.Accelerator.pad_across_processes` method to pad you tensor to the
+    biggest size across processes.
+
 
 Launching your distributed script
 -----------------------------------------------------------------------------------------------------------------------
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index d9f7cf1bc..75539559e 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -22,7 +22,7 @@
 from .kwargs_handlers import DistributedDataParallelKwargs, GradScalerKwargs, KwargsHandler
 from .optimizer import AcceleratedOptimizer
 from .state import AcceleratorState, DistributedType
-from .utils import RNGType, extract_model_from_parallel, gather, save, wait_for_everyone
+from .utils import RNGType, extract_model_from_parallel, gather, pad_across_processes, save, wait_for_everyone
 
 
 class Accelerator:
@@ -289,6 +289,23 @@ def gather(self, tensor):
         """
         return gather(tensor)
 
+    def pad_across_processes(self, tensor, dim=0, pad_index=0, pad_first=False):
+        """
+        Recursively pad the tensors in a nested list/tuple/dictionary of tensors from all devices to the same size so
+        they can safely be gathered.
+
+        Args:
+            tensor (nested list/tuple/dictionary of :obj:`torch.Tensor`):
+                The data to gather.
+            dim (:obj:`int`, `optional`, defaults to 0):
+                The dimension on which to pad.
+            pad_index (:obj:`int`, `optional`, defaults to 0):
+                The value with which to pad.
+            pad_first (:obj:`bool`, `optional`, defaults to :obj:`False`):
+                Whether to pad at the beginning or the end.
+        """
+        return pad_across_processes(tensor, dim=dim, pad_index=pad_index, pad_first=pad_first)
+
     def unwrap_model(self, model):
         """
         Unwraps the :obj:`model` from the additional layer possible added by :meth:`~accelerate.Accelerator.prepare`.
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index f3a9ae92a..96368ba7b 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -21,7 +21,6 @@
 import subprocess
 import sys
 from pathlib import Path
-from typing import Optional
 
 from accelerate.commands.config import LaunchConfig, default_config_file
 from accelerate.state import DistributedType
@@ -68,9 +67,7 @@ def launch_command_parser(subparsers=None):
     parser.add_argument(
         "--machine_rank", type=int, default=0, help="The rank of the machine on which this script is launched."
     )
-    parser.add_argument(
-        "--main_process_ip", type=str, default=None, help="The IP address of the machine of rank 0."
-    )
+    parser.add_argument("--main_process_ip", type=str, default=None, help="The IP address of the machine of rank 0.")
     parser.add_argument(
         "--main_process_port",
         type=int,
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 61a39fc95..e67a4cee1 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -148,7 +148,7 @@ def _gpu_gather(tensor):
 
 def gather(tensor):
     """
-    Recusrively gather tensor in a nested list/tuple/dictionary of tensors from all devices.
+    Recursively gather tensor in a nested list/tuple/dictionary of tensors from all devices.
 
     Args:
         tensor (nested list/tuple/dictionary of :obj:`torch.Tensor`):
@@ -165,6 +165,47 @@ def gather(tensor):
         return tensor
 
 
+def pad_across_processes(tensor, dim=0, pad_index=0, pad_first=False):
+    """
+    Recursively pad the tensors in a nested list/tuple/dictionary of tensors from all devices to the same size so they
+    can safely be gathered.
+
+    Args:
+        tensor (nested list/tuple/dictionary of :obj:`torch.Tensor`):
+            The data to gather.
+        dim (:obj:`int`, `optional`, defaults to 0):
+            The dimension on which to pad.
+        pad_index (:obj:`int`, `optional`, defaults to 0):
+            The value with which to pad.
+        pad_first (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether to pad at the beginning or the end.
+    """
+    if isinstance(tensor, (list, tuple)):
+        return type(tensor)(pad_across_processes(t, dim=dim, pad_index=pad_index) for t in tensor)
+    elif isinstance(tensor, dict):
+        return type(tensor)({k: pad_across_processes(v, dim=dim, pad_index=pad_index) for k, v in tensor.items()})
+    elif not isinstance(tensor, torch.Tensor):
+        raise TypeError(f"Can't pad the values of type {type(tensor)}, only of nested list/tuple/dicts of tensors.")
+
+    # Gather all sizes
+    size = torch.tensor(tensor.shape, device=tensor.device)[None]
+    sizes = gather(size).cpu()
+    # Then pad to the maximum size
+    max_size = max(s[dim] for s in sizes)
+    old_size = tensor.shape
+    new_size = list(old_size)
+    new_size[dim] = max_size
+    new_tensor = tensor.new_zeros(tuple(new_size)) + pad_index
+    if pad_first:
+        indices = tuple(
+            slice(max_size - old_size[dim], max_size) if i == dim else slice(None) for i in range(len(size))
+        )
+    else:
+        indices = tuple(slice(0, old_size[dim]) if i == dim else slice(None) for i in range(len(size)))
+    new_tensor[indices] = tensor
+    return new_tensor
+
+
 def wait_for_everyone():
     """
     Introduces a blocking point in the script, making sure all processes have reached this point before continuing.
diff --git a/tests/test_multigpu.py b/tests/test_multigpu.py
index ce69a3db3..7ffbd94f8 100644
--- a/tests/test_multigpu.py
+++ b/tests/test_multigpu.py
@@ -20,6 +20,7 @@
 import torch
 
 import accelerate
+from accelerate import Accelerator
 from accelerate.test_utils import execute_subprocess_async, require_multi_gpu
 
 
@@ -39,3 +40,43 @@ def test_multi_gpu(self):
         """.split()
         cmd = [sys.executable] + distributed_args
         execute_subprocess_async(cmd, env=os.environ.copy())
+
+    @require_multi_gpu
+    def test_pad_across_processes(self):
+        distributed_args = f"""
+            -m torch.distributed.launch
+            --nproc_per_node={torch.cuda.device_count()}
+            --use_env
+            {inspect.getfile(self.__class__)}
+        """.split()
+        cmd = [sys.executable] + distributed_args
+        execute_subprocess_async(cmd, env=os.environ.copy())
+
+
+if __name__ == "__main__":
+    accelerator = Accelerator()
+    shape = (accelerator.state.process_index + 2, 10)
+    tensor = torch.randint(0, 10, shape).to(accelerator.device)
+
+    error_msg = ""
+
+    tensor1 = accelerator.pad_across_processes(tensor)
+    if tensor1.shape[0] != accelerator.state.num_processes + 1:
+        error_msg += f"Found shape {tensor1.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
+    if not torch.equal(tensor1[: accelerator.state.process_index + 2], tensor):
+        error_msg += "Tensors have different values."
+    if not torch.all(tensor1[accelerator.state.process_index + 2 :] == 0):
+        error_msg += "Padding was not done with the right value (0)."
+
+    tensor2 = accelerator.pad_across_processes(tensor, pad_first=True)
+    if tensor2.shape[0] != accelerator.state.num_processes + 1:
+        error_msg += f"Found shape {tensor2.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
+    index = accelerator.state.num_processes - accelerator.state.process_index - 1
+    if not torch.equal(tensor2[index:], tensor):
+        error_msg += "Tensors have different values."
+    if not torch.all(tensor2[:index] == 0):
+        error_msg += "Padding was not done with the right value (0)."
+
+    # Raise error at the end to make sure we don't stop at the first failure.
+    if len(error_msg) > 0:
+        raise ValueError(error_msg)