diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 354089c38..e8ff21884 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -855,17 +855,23 @@ def gather(self, tensor):
         """
         return gather(tensor)
 
-    def reduce(self, tensor: torch.Tensor, reduction="sum"):
+    def reduce(self, tensor, reduction="sum"):
         """
         Reduce the values in *tensor* across all processes based on *reduction*.
 
+        Note:
+            All processes get the reduced value.
+
         Args:
-            tensor (`torch.Tensor`):
+            tensor (`torch.Tensor`, or a nested tuple/list/dictionary of `torch.Tensor`):
                 The tensors to reduce across all processes.
             reduction (`str`, *optional*, defaults to "sum"):
                 A reduction type, can be one of 'sum', 'mean', or 'none'. If 'none', will not perform any operation.
+
+        Returns:
+            `torch.Tensor`, or a nested tuple/list/dictionary of `torch.Tensor`: The reduced tensor(s).
         """
-        reduce(tensor, reduction)
+        return reduce(tensor, reduction)
 
     def pad_across_processes(self, tensor, dim=0, pad_index=0, pad_first=False):
         """
diff --git a/src/accelerate/utils/operations.py b/src/accelerate/utils/operations.py
index 46753513e..7505c04ce 100644
--- a/src/accelerate/utils/operations.py
+++ b/src/accelerate/utils/operations.py
@@ -430,7 +430,7 @@ def _reduce_across_processes(tensor, reduction="mean"):
             xm.all_reduce("sum", cloned_tensor)
             return cloned_tensor
         elif state.distributed_type in [DistributedType.DEEPSPEED, DistributedType.MULTI_GPU]:
-            torch.distributed.reduce(cloned_tensor, ReduceOp.SUM)
+            torch.distributed.all_reduce(cloned_tensor, ReduceOp.SUM)
             return cloned_tensor
         else:
             if reduction == "sum":



Discussion: body: Fix a bug when using reduce at a tensor.

Before:
```
a = torch.tensor(accelerator.process_index, device=accelerator.device)

accelerator.reduce(a, reduction='sum') # don't change a
# or
a = accelerator.reduce(a, reduction='sum') # a is None
```

After:
```
a = torch.tensor(accelerator.process_index, device=accelerator.device)
a = accelerator.reduce(a, reduction='sum') # a is correctly reduced
```

body: Thanks for the fix! 😄
body: _The documentation is not available anymore as the PR was closed or merged._
body: When I update the doc for this pr, I find that the `accelerate.utils.reduce` seems have inconsistent behavior on different type of devices.

In detail, it will reduce value on all processes, i.e. `all_reduce` when device is TPU. But only reduce to the main process, i.e. `reduce`, when device is GPU.

Could I change the behavior to `all_reduce` when device is GPU ?


For more infomation, see the doc stuff [here](https://github.com/wwhio/accelerate/blob/96a17aae1164da5511a50d7138bd6f8ae5303267/src/accelerate/accelerator.py#L862-L864) and the `accelerate.utils.reduce` code [here](https://github.com/wwhio/accelerate/blob/96a17aae1164da5511a50d7138bd6f8ae5303267/src/accelerate/utils/operations.py#L411-L441).

body: Yes, I think it should also be `all_reduce` on GPU to avoid bugs when not all processes have the same tensor later in the code.