diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index eb3b24ce9..f62d9e7a4 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -40,6 +40,7 @@
     gather,
     get_pretty_name,
     pad_across_processes,
+    reduce,
     save,
     wait_for_everyone,
 )
@@ -556,7 +557,7 @@ def clip_grad_value_(self, parameters, clip_value):
 
     def gather(self, tensor):
         """
-        Gather the values in *tensor* accross all processes and concatenate them on the first dimension. Useful to
+        Gather the values in *tensor* across all processes and concatenate them on the first dimension. Useful to
         regroup the predictions from all processes when doing evaluation.
 
         Note:
@@ -572,6 +573,18 @@ def gather(self, tensor):
         """
         return gather(tensor)
 
+    def reduce(self, tensor: torch.Tensor, reduction="sum"):
+        """
+        Reduce the values in *tensor* across all processes based on *reduction*.
+
+        Args:
+            tensor (`torch.Tensor`):
+                The tensors to reduce across all processes.
+            reduction (`str`, *optional*, defaults to "sum"):
+                A reduction type, can be one of 'sum', 'mean', or 'none'. If 'none', will not perform any operation.
+        """
+        reduce(tensor, reduction)
+
     def pad_across_processes(self, tensor, dim=0, pad_index=0, pad_first=False):
         """
         Recursively pad the tensors in a nested list/tuple/dictionary of tensors from all devices to the same size so
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 9bcd87f5f..c2d05c162 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -24,6 +24,7 @@
 
 import numpy as np
 import torch
+from torch.distributed import ReduceOp
 
 from packaging import version
 
@@ -621,6 +622,39 @@ def _pad_across_processes(tensor, dim=0, pad_index=0, pad_first=False):
     )
 
 
+def reduce(tensor, reduction="mean"):
+    """
+    Recursively reduce the tensors in a nested list/tuple/dictionary of lists of tensors across all processes by the
+    mean of a given operation.
+
+    Args:
+        tensor (nested list/tuple/dictionary of `torch.Tensor`):
+            The data to reduce.
+        reduction (`str`, *optional*, defaults to `"mean"`):
+            A reduction method. Can be of "mean", "sum", or "none"
+
+    Returns:
+        The same data structure as `data` with all the tensors reduced.
+    """
+
+    def _reduce_across_processes(tensor, reduction="mean"):
+        state = AcceleratorState()
+        cloned_tensor = tensor.clone()
+        if state.distributed_type == DistributedType.TPU:
+            xm.all_reduce("sum", cloned_tensor)
+            return cloned_tensor
+        elif state.distributed_type in [DistributedType.DEEPSPEED, DistributedType.MULTI_GPU]:
+            torch.distributed.reduce(cloned_tensor, ReduceOp.SUM)
+            return cloned_tensor
+        else:
+            if reduction == "sum":
+                return cloned_tensor.sum()
+            else:
+                return cloned_tensor.mean()
+
+    return recursively_apply(_reduce_across_processes, tensor, error_on_other_type=True, reduction=reduction)
+
+
 def wait_for_everyone():
     """
     Introduces a blocking point in the script, making sure all processes have reached this point before continuing.


Review comments: body: ```suggestion
        reduction (`str`, *optional*, defaults to `"mean"`):
```
diff_hunk: @@ -621,6 +622,35 @@ def _pad_across_processes(tensor, dim=0, pad_index=0, pad_first=False):
     )
 
 
+def reduce(tensor: torch.Tensor, reduction="mean"):
+    """
+    Reduce `tensor` across all processes and perform a `reduction`.
+
+    Args:
+        tensor (`torch.Tensor`):
+            The data to reduce
+        reduction (`str`, *optional*, defaults to "mean"):
body: ```suggestion
    Reduce a tensor across all processes by the mean of a given operation.
```
The arguments are described just below, so I like to keep the general description pure English :-)
diff_hunk: @@ -621,6 +622,35 @@ def _pad_across_processes(tensor, dim=0, pad_index=0, pad_first=False):
     )
 
 
+def reduce(tensor: torch.Tensor, reduction="mean"):
+    """
+    Reduce `tensor` across all processes and perform a `reduction`.

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._