diff --git a/src/accelerate/big_modeling.py b/src/accelerate/big_modeling.py
index e277e89a2..24857ad25 100644
--- a/src/accelerate/big_modeling.py
+++ b/src/accelerate/big_modeling.py
@@ -210,7 +210,8 @@ def dispatch_model(
     device_map: Dict[str, Union[str, int, torch.device]],
     main_device: Optional[torch.device] = None,
     state_dict: Optional[Dict[str, torch.Tensor]] = None,
-    offload_dir: Union[str, os.PathLike] = None,
+    offload_dir: Optional[Union[str, os.PathLike]] = None,
+    offload_index: Optional[Dict[str, str]] = None,
     offload_buffers: bool = False,
     preload_module_classes: Optional[List[str]] = None,
 ):
@@ -231,6 +232,9 @@ def dispatch_model(
             The state dict of the part of the model that will be kept on CPU.
         offload_dir (`str` or `os.PathLike`):
             The folder in which to offload the model weights (or where the model weights are already offloaded).
+        offload_index (`Dict`, *optional*):
+            A dictionary from weight name to their information (`dtype`/ `shape` or safetensors filename). Will default
+            to the index saved in `save_folder`.
         offload_buffers (`bool`, *optional*, defaults to `False`):
             Whether or not to offload the buffers with the model parameters.
         preload_module_classes (`List[str]`, *optional*):
@@ -256,13 +260,15 @@ def dispatch_model(
             state_dict = extract_submodules_state_dict(model.state_dict(), cpu_modules)
 
     disk_modules = [name for name, device in device_map.items() if device == "disk"]
-    if offload_dir is None and len(disk_modules) > 0:
+    if offload_dir is None and offload_index is None and len(disk_modules) > 0:
         raise ValueError(
             "We need an `offload_dir` to dispatch this model according to this `device_map`, the following submodules "
             f"need to be offloaded: {', '.join(disk_modules)}."
         )
-    if len(disk_modules) > 0 and (
-        not os.path.isdir(offload_dir) or not os.path.isfile(os.path.join(offload_dir, "index.json"))
+    if (
+        len(disk_modules) > 0
+        and offload_index is None
+        and (not os.path.isdir(offload_dir) or not os.path.isfile(os.path.join(offload_dir, "index.json")))
     ):
         disk_state_dict = extract_submodules_state_dict(model.state_dict(), disk_modules)
         offload_state_dict(offload_dir, disk_state_dict)
@@ -273,8 +279,11 @@ def dispatch_model(
     offloaded_devices = ["disk"] if main_device == "cpu" else ["cpu", "disk"]
     offload = {name: device in offloaded_devices for name, device in device_map.items()}
     save_folder = offload_dir if len(disk_modules) > 0 else None
-    if state_dict is not None or save_folder is not None:
-        weights_map = OffloadedWeightsLoader(state_dict=state_dict, save_folder=save_folder)
+    if state_dict is not None or save_folder is not None or offload_index is not None:
+        device = main_device if offload_index is not None else None
+        weights_map = OffloadedWeightsLoader(
+            state_dict=state_dict, save_folder=save_folder, index=offload_index, device=device
+        )
     else:
         weights_map = None
 
diff --git a/src/accelerate/utils/__init__.py b/src/accelerate/utils/__init__.py
index 87f891021..f46bce267 100644
--- a/src/accelerate/utils/__init__.py
+++ b/src/accelerate/utils/__init__.py
@@ -34,6 +34,7 @@
     is_megatron_lm_available,
     is_mlflow_available,
     is_rich_available,
+    is_safetensors_available,
     is_sagemaker_available,
     is_tensorboard_available,
     is_tpu_available,
diff --git a/src/accelerate/utils/imports.py b/src/accelerate/utils/imports.py
index bfedee162..72d6202cd 100644
--- a/src/accelerate/utils/imports.py
+++ b/src/accelerate/utils/imports.py
@@ -101,6 +101,10 @@ def is_megatron_lm_available():
     return False
 
 
+def is_safetensors_available():
+    return importlib.util.find_spec("safetensors") is not None
+
+
 def is_transformers_available():
     return importlib.util.find_spec("transformers") is not None
 
diff --git a/src/accelerate/utils/offload.py b/src/accelerate/utils/offload.py
index bc0fcdef1..22f51cce1 100644
--- a/src/accelerate/utils/offload.py
+++ b/src/accelerate/utils/offload.py
@@ -20,6 +20,12 @@
 import numpy as np
 import torch
 
+from ..logging import get_logger
+from .imports import is_safetensors_available
+
+
+logger = get_logger(__name__)
+
 
 def offload_weight(weight, weight_name, offload_folder, index=None):
     dtype = None
@@ -131,8 +137,8 @@ class OffloadedWeightsLoader(Mapping):
         save_folder (`str` or `os.PathLike`, *optional*):
             The directory in which the weights are stored (by `offload_state_dict` for instance).
         index (`Dict`, *optional*):
-            A dictionary from weight name to their information (`dtype` and `shape`). Will default to the index saved
-            in `save_folder`.
+            A dictionary from weight name to their information (`dtype`/ `shape` or safetensors filename). Will default
+            to the index saved in `save_folder`.
     """
 
     def __init__(
@@ -140,6 +146,7 @@ def __init__(
         state_dict: Dict[str, torch.Tensor] = None,
         save_folder: Optional[Union[str, os.PathLike]] = None,
         index: Mapping = None,
+        device=None,
     ):
         if state_dict is None and save_folder is None:
             raise ValueError("Need either a `state_dict` or a `save_folder` containing offloaded weights.")
@@ -152,12 +159,32 @@ def __init__(
         self.index = {} if index is None else index
         self.all_keys = list(self.state_dict.keys())
         self.all_keys.extend([key for key in self.index if key not in self.all_keys])
+        self.device = device
 
     def __getitem__(self, key: str):
         # State dict gets priority
         if key in self.state_dict:
             return self.state_dict[key]
         weight_info = self.index[key]
+        if weight_info.get("safetensors_file") is not None:
+            if not is_safetensors_available():
+                raise ImportError("These offloaded weights require the use of safetensors: `pip install safetensors`.")
+
+            if "SAFETENSORS_FAST_GPU" not in os.environ:
+                logger.info("Enabling fast loading with safetensors by setting `SAFETENSORS_FAST_GPU` to 1.")
+                os.environ["SAFETENSORS_FAST_GPU"] = "1"
+
+            from safetensors import safe_open
+
+            device = "cpu" if self.device is None else self.device
+            with safe_open(weight_info["safetensors_file"], framework="pt", device=device) as f:
+                tensor = f.get_tensor(weight_info.get("weight_name", key))
+
+            if "dtype" in weight_info:
+                return tensor.to(getattr(torch, weight_info["dtype"]))
+            else:
+                return tensor
+
         weight_file = os.path.join(self.save_folder, f"{key}.dat")
         return load_offloaded_weight(weight_file, weight_info)
 


Review comments: body: We need a guard here for `is_safetensors_available` and raise an error if not :) (aka make one :) )
diff_hunk: @@ -152,12 +153,25 @@ def __init__(
         self.index = {} if index is None else index
         self.all_keys = list(self.state_dict.keys())
         self.all_keys.extend([key for key in self.index if key not in self.all_keys])
+        self.device = device
 
     def __getitem__(self, key: str):
         # State dict gets priority
         if key in self.state_dict:
             return self.state_dict[key]
         weight_info = self.index[key]
+        if weight_info.get("safetensors_file") is not None:
+            from safetensors import safe_open
body: Indeed, will clean that up on Monday!
diff_hunk: @@ -152,12 +153,25 @@ def __init__(
         self.index = {} if index is None else index
         self.all_keys = list(self.state_dict.keys())
         self.all_keys.extend([key for key in self.index if key not in self.all_keys])
+        self.device = device
 
     def __getitem__(self, key: str):
         # State dict gets priority
         if key in self.state_dict:
             return self.state_dict[key]
         weight_info = self.index[key]
+        if weight_info.get("safetensors_file") is not None:
+            from safetensors import safe_open

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._