diff --git a/src/accelerate/utils/__init__.py b/src/accelerate/utils/__init__.py
index 60f39c90a..309b49023 100644
--- a/src/accelerate/utils/__init__.py
+++ b/src/accelerate/utils/__init__.py
@@ -48,6 +48,7 @@
     OffloadedWeightsLoader,
     PrefixedDataset,
     extract_submodules_state_dict,
+    load_offloaded_weight,
     offload_state_dict,
     offload_weight,
     save_offload_index,
diff --git a/src/accelerate/utils/offload.py b/src/accelerate/utils/offload.py
index a562e9283..6e8b34bbc 100644
--- a/src/accelerate/utils/offload.py
+++ b/src/accelerate/utils/offload.py
@@ -22,10 +22,18 @@
 
 
 def offload_weight(weight, weight_name, offload_folder, index=None):
+    dtype = None
+    # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
+    if str(weight.dtype) == "torch.bfloat16":
+        # Need to convert to FP32 since NumPy does not handle bfloat16s.
+        weight = weight.float()
+        dtype = "bfloat16"
     array = weight.numpy()
     tensor_file = os.path.join(offload_folder, f"{weight_name}.dat")
     if index is not None:
-        index[weight_name] = {"dtype": str(array.dtype), "shape": list(array.shape)}
+        if dtype is None:
+            dtype = str(array.dtype)
+        index[weight_name] = {"dtype": dtype, "shape": list(array.shape)}
     if array.ndim == 0:
         array = array[None]
     file_array = np.memmap(tensor_file, dtype=array.dtype, mode="w+", shape=array.shape)
@@ -34,6 +42,28 @@ def offload_weight(weight, weight_name, offload_folder, index=None):
     return index
 
 
+def load_offloaded_weight(weight_file, weight_info):
+    shape = tuple(weight_info["shape"])
+    if shape == ():
+        # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
+        shape = (1,)
+
+    dtype = weight_info["dtype"]
+    if dtype == "bfloat16":
+        # NumPy does not support bfloat16 so this was saved as a float32
+        dtype = "float32"
+
+    weight = np.memmap(weight_file, dtype=dtype, shape=shape, mode="r")
+
+    if len(weight_info["shape"]) == 0:
+        weight = weight[0]
+    weight = torch.tensor(weight)
+    if weight_info["dtype"] == "bfloat16":
+        weight = weight.to(torch.bfloat16)
+
+    return weight
+
+
 def save_offload_index(index, offload_folder):
     if index is None or len(index) == 0:
         # Nothing to save
@@ -129,12 +159,7 @@ def __getitem__(self, key: str):
             return self.state_dict[key]
         weight_info = self.index[key]
         weight_file = os.path.join(self.save_folder, f"{key}.dat")
-        shape = tuple(weight_info["shape"])
-        if shape == ():
-            weight = np.memmap(weight_file, dtype=weight_info["dtype"], shape=(1,), mode="r")[0]
-        else:
-            weight = np.memmap(weight_file, dtype=weight_info["dtype"], shape=shape, mode="r")
-        return torch.tensor(weight)
+        return load_offloaded_weight(weight_file, weight_info)
 
     def __iter__(self):
         return iter(self.all_keys)
diff --git a/tests/test_offload.py b/tests/test_offload.py
index 515d5094d..765a9685d 100644
--- a/tests/test_offload.py
+++ b/tests/test_offload.py
@@ -19,7 +19,13 @@
 import torch
 import torch.nn as nn
 
-from accelerate.utils import OffloadedWeightsLoader, offload_state_dict
+from accelerate.utils import (
+    OffloadedWeightsLoader,
+    is_torch_version,
+    load_offloaded_weight,
+    offload_state_dict,
+    offload_weight,
+)
 
 
 class ModelForTest(nn.Module):
@@ -35,8 +41,6 @@ def forward(self, x):
 
 class OffloadTester(unittest.TestCase):
     def test_offload_state_dict(self):
-        from tempfile import TemporaryDirectory
-
         model = ModelForTest()
         with TemporaryDirectory() as tmp_dir:
             offload_state_dict(tmp_dir, model.state_dict())
@@ -49,6 +53,22 @@ def test_offload_state_dict(self):
                 self.assertTrue(os.path.isfile(weight_file))
                 # TODO: add tests on the fact weights are properly loaded
 
+    def test_offload_weight(self):
+        dtypes = [torch.float16, torch.float32]
+        if is_torch_version(">=", "1.10"):
+            dtypes.append(torch.bfloat16)
+
+        for dtype in dtypes:
+            weight = torch.randn(2, 3, dtype=dtype)
+            with TemporaryDirectory() as tmp_dir:
+                index = offload_weight(weight, "weight", tmp_dir, {})
+                weight_file = os.path.join(tmp_dir, "weight.dat")
+                self.assertTrue(os.path.isfile(weight_file))
+                self.assertDictEqual(index, {"weight": {"shape": [2, 3], "dtype": str(dtype).split(".")[1]}})
+
+                new_weight = load_offloaded_weight(weight_file, index["weight"])
+                self.assertTrue(torch.equal(weight, new_weight))
+
     def test_offload_weights_loader(self):
         model = ModelForTest()
         state_dict = model.state_dict()


Review comments: body: Since we care about the prevision here, would it be a good idea to test these with floats that marginally get close to the float16 reach to ensure they're lossless? 

E.g.:
```python
weight = torch.rand((2,3), dtype=dtype)
```


diff_hunk: @@ -49,6 +53,22 @@ def test_offload_state_dict(self):
                 self.assertTrue(os.path.isfile(weight_file))
                 # TODO: add tests on the fact weights are properly loaded
 
+    def test_offload_weight(self):
+        dtypes = [torch.float16, torch.float32]
+        if is_torch_version(">=", "1.10"):
+            dtypes.append(torch.bfloat16)
+
+        for dtype in dtypes:
+            weight = torch.tensor([[1, 2, 3], [4, 5, 6]], dtype=dtype)
body: Probably yes 😅 
diff_hunk: @@ -49,6 +53,22 @@ def test_offload_state_dict(self):
                 self.assertTrue(os.path.isfile(weight_file))
                 # TODO: add tests on the fact weights are properly loaded
 
+    def test_offload_weight(self):
+        dtypes = [torch.float16, torch.float32]
+        if is_torch_version(">=", "1.10"):
+            dtypes.append(torch.bfloat16)
+
+        for dtype in dtypes:
+            weight = torch.tensor([[1, 2, 3], [4, 5, 6]], dtype=dtype)

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._