diff --git a/src/accelerate/__init__.py b/src/accelerate/__init__.py
index d22006d54..9129fc1a5 100644
--- a/src/accelerate/__init__.py
+++ b/src/accelerate/__init__.py
@@ -5,7 +5,14 @@
 __version__ = "0.15.0.dev0"
 
 from .accelerator import Accelerator
-from .big_modeling import cpu_offload, disk_offload, dispatch_model, init_empty_weights, load_checkpoint_and_dispatch
+from .big_modeling import (
+    cpu_offload,
+    disk_offload,
+    dispatch_model,
+    init_empty_weights,
+    init_on_device,
+    load_checkpoint_and_dispatch,
+)
 from .launchers import debug_launcher, notebook_launcher
 from .utils import (
     DeepSpeedPlugin,
diff --git a/src/accelerate/big_modeling.py b/src/accelerate/big_modeling.py
index 24857ad25..d3247a464 100644
--- a/src/accelerate/big_modeling.py
+++ b/src/accelerate/big_modeling.py
@@ -62,6 +62,31 @@ def init_empty_weights(include_buffers: bool = False):
     """
     if not is_torch_version(">=", "1.9.0"):
         raise NotImplementedError("Initializing empty weights to a meta device requires torch >= 1.9.0")
+    with init_on_device(torch.device("meta"), include_buffers=include_buffers) as f:
+        yield f
+
+
+@contextmanager
+def init_on_device(device: torch.device, include_buffers: bool = False):
+    """
+    A context manager under which models are initialized with all parameters on the specified device.
+
+    Args:
+        device (`torch.device`):
+            Device to initialize all parameters on.
+        include_buffers (`bool`, *optional*, defaults to `False`):
+            Whether or not to also put all buffers on the meta device while initializing.
+
+    Example:
+
+    ```python
+    import torch.nn as nn
+    from accelerate import init_on_device
+
+    with init_on_device(device=torch.device("cuda")):
+        tst = nn.Liner(100, 100)  # on `cuda` device
+    ```
+    """
     old_register_parameter = nn.Module.register_parameter
     if include_buffers:
         old_register_buffer = nn.Module.register_buffer
@@ -71,12 +96,12 @@ def register_empty_parameter(module, name, param):
         if param is not None:
             param_cls = type(module._parameters[name])
             kwargs = module._parameters[name].__dict__
-            module._parameters[name] = param_cls(module._parameters[name].to(torch.device("meta")), **kwargs)
+            module._parameters[name] = param_cls(module._parameters[name].to(device), **kwargs)
 
     def register_empty_buffer(module, name, buffer):
         old_register_buffer(module, name, buffer)
         if buffer is not None:
-            module._buffers[name] = module._buffers[name].to(torch.device("meta"))
+            module._buffers[name] = module._buffers[name].to(device)
 
     # Patch tensor creation
     if include_buffers:
@@ -89,7 +114,7 @@ def register_empty_buffer(module, name, buffer):
 
     def patch_tensor_constructor(fn):
         def wrapper(*args, **kwargs):
-            kwargs["device"] = torch.device("meta")
+            kwargs["device"] = device
             return fn(*args, **kwargs)
 
         return wrapper
diff --git a/src/accelerate/test_utils/__init__.py b/src/accelerate/test_utils/__init__.py
index faf8a7da8..b58b932b1 100644
--- a/src/accelerate/test_utils/__init__.py
+++ b/src/accelerate/test_utils/__init__.py
@@ -8,6 +8,7 @@
     require_cpu,
     require_cuda,
     require_huggingface_suite,
+    require_mps,
     require_multi_gpu,
     require_single_gpu,
     require_torch_min_version,
diff --git a/src/accelerate/test_utils/testing.py b/src/accelerate/test_utils/testing.py
index 94e13412a..3a681bf32 100644
--- a/src/accelerate/test_utils/testing.py
+++ b/src/accelerate/test_utils/testing.py
@@ -87,6 +87,15 @@ def require_cuda(test_case):
     return unittest.skipUnless(torch.cuda.is_available(), "test requires a GPU")(test_case)
 
 
+def require_mps(test_case):
+    """
+    Decorator marking a test that requires MPS backend. These tests are skipped when torch doesn't support `mps`
+    backend.
+    """
+    is_mps_supported = hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
+    return unittest.skipUnless(is_mps_supported, "test requires a `mps` backend support in `torch`")(test_case)
+
+
 def require_huggingface_suite(test_case):
     """
     Decorator marking a test that requires transformers and datasets. These tests are skipped when they are not.
diff --git a/tests/test_big_modeling.py b/tests/test_big_modeling.py
index 4f738f4d8..467a91424 100644
--- a/tests/test_big_modeling.py
+++ b/tests/test_big_modeling.py
@@ -24,10 +24,11 @@
     disk_offload,
     dispatch_model,
     init_empty_weights,
+    init_on_device,
     load_checkpoint_and_dispatch,
 )
 from accelerate.hooks import remove_hook_from_submodules
-from accelerate.test_utils import require_cuda, require_multi_gpu, require_torch_min_version, slow
+from accelerate.test_utils import require_cuda, require_mps, require_multi_gpu, require_torch_min_version, slow
 from accelerate.utils import offload_state_dict
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
@@ -109,6 +110,22 @@ def test_init_empty_weights_very_large_model(self):
         with init_empty_weights():
             _ = nn.Sequential(*[nn.Linear(10000, 10000) for _ in range(1000)])
 
+    @require_cuda
+    def test_init_on_device_cuda(self):
+        device = torch.device("cuda:0")
+        with init_on_device(device):
+            model = nn.Linear(10, 10)
+        self.assertEqual(model.weight.device, device)
+        self.assertEqual(model.weight.device, device)
+
+    @require_mps
+    def test_init_on_device_mps(self):
+        device = torch.device("mps:0")
+        with init_on_device(device):
+            model = nn.Linear(10, 10)
+        self.assertEqual(model.weight.device, device)
+        self.assertEqual(model.weight.device, device)
+
     def test_cpu_offload(self):
         model = ModelForTest()
         x = torch.randn(2, 3)


Review comments: body: Ah ah, that example needs adapting ;-p

diff_hunk: @@ -62,6 +62,32 @@ def init_empty_weights(include_buffers: bool = False):
     """
     if not is_torch_version(">=", "1.9.0"):
         raise NotImplementedError("Initializing empty weights to a meta device requires torch >= 1.9.0")
+    with init_on_device(torch.device("meta"), include_buffers=include_buffers) as f:
+        yield f
+
+
+@contextmanager
+def init_on_device(device: torch.device, include_buffers: bool = False):
+    """
+    A context manager under which models are initialized with all parameters on the specified device.
+
+    Args:
+        device (`torch.device`):
+            Device to initialize all parameters on.
+        include_buffers (`bool`, *optional*, defaults to `False`):
+            Whether or not to also put all buffers on the meta device while initializing.
+
+    Example:
+
+    ```python
+    import torch.nn as nn
+    from accelerate import init_empty_weights
+
+    # Initialize a model with 100 billions parameters in no time and without using any RAM.
+    with init_on_device(device=torch.device("cuda")):
+        tst = nn.Liner(100, 100)  # on `cuda` device

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: I guess we can also extend to specific dtypes as well now.
body: Which dtype did you have in mind? For floating types there is a default you can set in PyTorch directly.
body: You mean https://pytorch.org/docs/stable/generated/torch.set_default_dtype.html ? Does fp16/bf16 also work? If so my bad.
body: Yes it works pretty well. We use it in Transformers [here](https://github.com/huggingface/transformers/blob/17292440c069118fbdb992b9a17da2098fab5b87/src/transformers/modeling_utils.py#L1109) for instance.