diff --git a/src/accelerate/big_modeling.py b/src/accelerate/big_modeling.py
index 4014e3972..85b1f3237 100644
--- a/src/accelerate/big_modeling.py
+++ b/src/accelerate/big_modeling.py
@@ -65,7 +65,9 @@ def init_empty_weights(include_buffers: bool = False):
     def register_empty_parameter(module, name, param):
         old_register_parameter(module, name, param)
         if param is not None:
-            module._parameters[name] = nn.Parameter(module._parameters[name].to(torch.device("meta")))
+            param_cls = type(module._parameters[name])
+            kwargs = module._parameters[name].__dict__
+            module._parameters[name] = param_cls(module._parameters[name].to(torch.device("meta")), **kwargs)
 
     def register_empty_buffer(module, name, buffer):
         old_register_buffer(module, name, buffer)
diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py
index 62c4f9761..2dc85ef70 100644
--- a/src/accelerate/utils/modeling.py
+++ b/src/accelerate/utils/modeling.py
@@ -124,11 +124,16 @@ def set_module_tensor_to_device(
             new_value = value.to(device)
         else:
             new_value = torch.tensor(value, device=device)
-    if is_buffer:
-        module._buffers[tensor_name] = new_value
-    else:
-        new_value = nn.Parameter(new_value, requires_grad=old_value.requires_grad)
-        module._parameters[tensor_name] = new_value
+
+        if is_buffer:
+            module._buffers[tensor_name] = new_value
+        elif module._parameters[tensor_name].device.type != "cuda":
+            param_cls = type(module._parameters[tensor_name])
+            kwargs = module._parameters[tensor_name].__dict__
+            new_value = param_cls(new_value, requires_grad=old_value.requires_grad, **kwargs).to(device)
+            module._parameters[tensor_name] = new_value
+        else:
+            module._parameters[tensor_name] = module._parameters[tensor_name].to(device)
 
 
 def named_module_tensors(module: nn.Module, include_buffers: bool = True, recurse: bool = False):


Review comments: body: This should stay at the previous level of indentation. To make sure there is absolutely zero breaking change, how about doing an `else`  with just `module._parameters[tensor_name].to(device)` ?
diff_hunk: @@ -124,11 +124,14 @@ def set_module_tensor_to_device(
             new_value = value.to(device)
         else:
             new_value = torch.tensor(value, device=device)
-    if is_buffer:
-        module._buffers[tensor_name] = new_value
-    else:
-        new_value = nn.Parameter(new_value, requires_grad=old_value.requires_grad)
-        module._parameters[tensor_name] = new_value
+
+        if is_buffer:
+            module._buffers[tensor_name] = new_value
+        elif module._parameters[tensor_name].device.type != "cuda":
+            param_cls = type(module._parameters[tensor_name])
+            kwargs = module._parameters[tensor_name].__dict__
+            new_value = param_cls(new_value, requires_grad=old_value.requires_grad, **kwargs).to(device)
+            module._parameters[tensor_name] = new_value
body: Just added it and seems to work fine! 🚀 
Can we wait a bit before merging the PR?
diff_hunk: @@ -124,11 +124,14 @@ def set_module_tensor_to_device(
             new_value = value.to(device)
         else:
             new_value = torch.tensor(value, device=device)
-    if is_buffer:
-        module._buffers[tensor_name] = new_value
-    else:
-        new_value = nn.Parameter(new_value, requires_grad=old_value.requires_grad)
-        module._parameters[tensor_name] = new_value
+
+        if is_buffer:
+            module._buffers[tensor_name] = new_value
+        elif module._parameters[tensor_name].device.type != "cuda":
+            param_cls = type(module._parameters[tensor_name])
+            kwargs = module._parameters[tensor_name].__dict__
+            new_value = param_cls(new_value, requires_grad=old_value.requires_grad, **kwargs).to(device)
+            module._parameters[tensor_name] = new_value
body: Sure thing, let me know when it's good for you. Also don't forget to run `make style` on your branch.
diff_hunk: @@ -124,11 +124,14 @@ def set_module_tensor_to_device(
             new_value = value.to(device)
         else:
             new_value = torch.tensor(value, device=device)
-    if is_buffer:
-        module._buffers[tensor_name] = new_value
-    else:
-        new_value = nn.Parameter(new_value, requires_grad=old_value.requires_grad)
-        module._parameters[tensor_name] = new_value
+
+        if is_buffer:
+            module._buffers[tensor_name] = new_value
+        elif module._parameters[tensor_name].device.type != "cuda":
+            param_cls = type(module._parameters[tensor_name])
+            kwargs = module._parameters[tensor_name].__dict__
+            new_value = param_cls(new_value, requires_grad=old_value.requires_grad, **kwargs).to(device)
+            module._parameters[tensor_name] = new_value
body: Great thanks a lot!!
diff_hunk: @@ -124,11 +124,14 @@ def set_module_tensor_to_device(
             new_value = value.to(device)
         else:
             new_value = torch.tensor(value, device=device)
-    if is_buffer:
-        module._buffers[tensor_name] = new_value
-    else:
-        new_value = nn.Parameter(new_value, requires_grad=old_value.requires_grad)
-        module._parameters[tensor_name] = new_value
+
+        if is_buffer:
+            module._buffers[tensor_name] = new_value
+        elif module._parameters[tensor_name].device.type != "cuda":
+            param_cls = type(module._parameters[tensor_name])
+            kwargs = module._parameters[tensor_name].__dict__
+            new_value = param_cls(new_value, requires_grad=old_value.requires_grad, **kwargs).to(device)
+            module._parameters[tensor_name] = new_value

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._