diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 3af792f1f..36eaea592 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -18,7 +18,7 @@
 from collections.abc import Mapping
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import List, Optional, Union
+from typing import Any, List, Optional, Union
 
 import numpy as np
 import torch
@@ -350,6 +350,39 @@ def gather(tensor):
         return tensor
 
 
+def _gpu_gather_object(object: Any):
+    def _gpu_gather_object_one(object: Any):
+        output_objects = [None for _ in range(AcceleratorState().num_processes)]
+        torch.distributed.all_gather_object(output_objects, object)
+        return output_objects
+
+    return recursively_apply(_gpu_gather_object_one, object)
+
+
+_cpu_gather_object = _gpu_gather_object
+
+
+def gather_object(object: Any):
+    """
+    Recursively gather object in a nested list/tuple/dictionary of objects from all devices.
+
+    Args:
+        object (nested list/tuple/dictionary of picklable object):
+            The data to gather.
+
+    Returns:
+        The same data structure as :obj:`object` with all the objects sent to every device.
+    """
+    if AcceleratorState().distributed_type == DistributedType.TPU:
+        raise NotImplementedError("gather objects in TPU is not supported")
+    elif AcceleratorState().distributed_type == DistributedType.MULTI_GPU:
+        return _gpu_gather_object(object)
+    elif AcceleratorState().distributed_type == DistributedType.MULTI_CPU:
+        return _cpu_gather_object(object)
+    else:
+        return object
+
+
 def _gpu_broadcast(data, src=0):
     def _gpu_broadcast_one(tensor, src=0):
         torch.distributed.broadcast(tensor, src=src)


Review comments: body: ```suggestion
        object (nested list/tuple/dictionary of any object):
```
diff_hunk: @@ -345,6 +345,40 @@ def gather(tensor):
         return tensor
 
 
+
+def _gpu_gather_object(object: Any):
+    def _gpu_gather_object_one(object: Any):
+        output_objects = [None for _ in range(torch.distributed.get_world_size())]
+        torch.distributed.all_gather_object(output_objects, object)
+        return [i for j in output_objects for i in j]
+
+    return recursively_apply(_gpu_gather_one, tensor, error_on_other_type=True)
+
+
+_cpu_gather_object = _gpu_gather_object
+
+
+def gather_object(object: Any):
+    """
+    Recursively gather object in a nested list/tuple/dictionary of objects from all devices.
+
+    Args:
+        object (nested list/tuple/dictionary of :obj:`Any`):
body: Here we should use the `state.num_processes` instead.
diff_hunk: @@ -345,6 +345,40 @@ def gather(tensor):
         return tensor
 
 
+
+def _gpu_gather_object(object: Any):
+    def _gpu_gather_object_one(object: Any):
+        output_objects = [None for _ in range(torch.distributed.get_world_size())]
body: I don't understand this line. This presupposes that every element in `output_objects` is iterable, but `object` is supposed to be a primitive object at this point (tensor or other if we remove `error_on_type=True`).
diff_hunk: @@ -345,6 +345,40 @@ def gather(tensor):
         return tensor
 
 
+
+def _gpu_gather_object(object: Any):
+    def _gpu_gather_object_one(object: Any):
+        output_objects = [None for _ in range(torch.distributed.get_world_size())]
+        torch.distributed.all_gather_object(output_objects, object)
+        return [i for j in output_objects for i in j]
body: `torch.all_gather_object` works for all picklable objects, so we can skip the `error_on_other_type=True` I believe.
diff_hunk: @@ -345,6 +345,40 @@ def gather(tensor):
         return tensor
 
 
+
+def _gpu_gather_object(object: Any):
+    def _gpu_gather_object_one(object: Any):
+        output_objects = [None for _ in range(torch.distributed.get_world_size())]
+        torch.distributed.all_gather_object(output_objects, object)
+        return [i for j in output_objects for i in j]
+
+    return recursively_apply(_gpu_gather_one, tensor, error_on_other_type=True)
body: ```suggestion
        The same data structure as :obj:`object` with all the objects sent to every device.
```
I think we need to explain where the list arrives (since each tensor is replaced by a list of tensors normally).
diff_hunk: @@ -345,6 +345,40 @@ def gather(tensor):
         return tensor
 
 
+
+def _gpu_gather_object(object: Any):
+    def _gpu_gather_object_one(object: Any):
+        output_objects = [None for _ in range(torch.distributed.get_world_size())]
+        torch.distributed.all_gather_object(output_objects, object)
+        return [i for j in output_objects for i in j]
+
+    return recursively_apply(_gpu_gather_one, tensor, error_on_other_type=True)
+
+
+_cpu_gather_object = _gpu_gather_object
+
+
+def gather_object(object: Any):
+    """
+    Recursively gather object in a nested list/tuple/dictionary of objects from all devices.
+
+    Args:
+        object (nested list/tuple/dictionary of :obj:`Any`):
+            The data to gather.
+
+    Returns:
+        The same data structure as :obj:`Any` with all objects sent to the proper device.
body: ```suggestion
        output_objects = [None for _ in range(AcceleratorState().num_processes)]
```
`state` is not defined.
diff_hunk: @@ -350,6 +350,39 @@ def gather(tensor):
         return tensor
 
 
+def _gpu_gather_object(object: Any):
+    def _gpu_gather_object_one(object: Any):
+        output_objects = [None for _ in range(state.num_processes)]
body: ```suggestion
    return recursively_apply(_gpu_gather_object_one, object)
```
diff_hunk: @@ -350,6 +350,39 @@ def gather(tensor):
         return tensor
 
 
+def _gpu_gather_object(object: Any):
+    def _gpu_gather_object_one(object: Any):
+        output_objects = [None for _ in range(state.num_processes)]
+        torch.distributed.all_gather_object(output_objects, object)
+        return output_objects
+
+    return recursively_apply(_gpu_gather_one, tensor)

Discussion: body: > I don't understand this line. This presupposes that every element in output_objects is iterable, but object is supposed to be a primitive object at this point (tensor or other if we remove error_on_type=True).

Should we separate `gather_object` to two functions, `gather_object_list` perform `[i for j in output_objects for i in j]` so it works much like original `gather`, while `gather_object` doesn not perform concatenation so it has better support for every objects

Or maybe we could add an argument to control it's behavior like

```python
def _gpu_gather_object(object: Any, concat: bool = True):
    def _gpu_gather_object_one(object: Any):
        output_objects = [None for _ in range(torch.distributed.get_world_size())]
        torch.distributed.all_gather_object(output_objects, object)
        if concat:
            return [i for j in output_objects for i in j]
        else:
            return output_objects 
```
body: It can't really concatenate if the tensors are not of the same size, so `gather_object` should not concatenate IMO.
body: > It can't really concatenate if the tensors are not of the same size, so `gather_object` should not concatenate IMO.

Yes we can leave it to the user to concat manually. 
body: I have resolved all conversations, could you please review it again?
body: > Thanks for addressing all comments, this looks good to merge! Could you just run `make style` on your branch to fix the quality failure?

Sorry for this late reply.... I believe I've resolved the quality issue.
body: > Here is how to solve the last quality issues (variables not defined).

Thank you for your kind suggestions! I have resolved these
(Sorry I'm recently experiencing some computer issues and `make style` is not giving me full outputs. 
body: Hmmmm, is it possible for us to determine the type of input data and auto call corresponding gather function? I just noticed [torchvison](https://github.com/pytorch/vision/blob/main/references/detection/utils.py#L70-L83) uses `gather_object` at all time
body: That's too magical for Accelerate's philosophy, users should pick which function they want to use.

All green now so we can merge your PR :-) Thanks again!
body: > That's too magical for Accelerate's philosophy, users should pick which function they want to use.

Haha, thats the reason why i love this repo so much. Users should be trained programmer and should know what they're doing/supposed to do. 

> All green now so we can merge your PR :-) Thanks again!

Thank you for all the review and comments~