diff --git a/src/accelerate/big_modeling.py b/src/accelerate/big_modeling.py
index 9d57c144f..83c6738af 100644
--- a/src/accelerate/big_modeling.py
+++ b/src/accelerate/big_modeling.py
@@ -143,6 +143,8 @@ def cpu_offload(
         execution_device = next(iter(model.parameters())).device
     if state_dict is None:
         state_dict = {n: p.to("cpu") for n, p in model.state_dict().items()}
+
+    add_hook_to_module(model, AlignDevicesHook(io_same_device=True), append=True)
     attach_align_device_hook(
         model,
         execution_device=execution_device,
@@ -151,7 +153,7 @@ def cpu_offload(
         weights_map=state_dict,
         preload_module_classes=preload_module_classes,
     )
-    add_hook_to_module(model, AlignDevicesHook(io_same_device=True))
+
     return model
 
 
@@ -189,6 +191,8 @@ def disk_offload(
     if execution_device is None:
         execution_device = next(iter(model.parameters())).device
     weights_map = OffloadedWeightsLoader(save_folder=offload_dir)
+
+    add_hook_to_module(model, AlignDevicesHook(io_same_device=True), append=True)
     attach_align_device_hook(
         model,
         execution_device=execution_device,
@@ -197,7 +201,7 @@ def disk_offload(
         weights_map=weights_map,
         preload_module_classes=preload_module_classes,
     )
-    add_hook_to_module(model, AlignDevicesHook(io_same_device=True))
+
     return model
 
 
diff --git a/src/accelerate/hooks.py b/src/accelerate/hooks.py
index 493444daa..120bb7f42 100644
--- a/src/accelerate/hooks.py
+++ b/src/accelerate/hooks.py
@@ -108,26 +108,34 @@ def detach_hook(self, module):
         return module
 
 
-def add_hook_to_module(module: nn.Module, hook: ModelHook):
+def add_hook_to_module(module: nn.Module, hook: ModelHook, append: bool = False):
     """
     Adds a hook to a given module. This will rewrite the `forward` method of the module to include the hook, to remove
     this behavior and restore the original `forward` method, use `remove_hook_from_module`.
 
     <Tip warning={true}>
 
-    If the module already contains a hook, this will replace it with the new hook passed. To chain two hooks together,
-    use the `SequentialHook` class.
+    If the module already contains a hook, this will replace it with the new hook passed by default. To chain two hooks
+    together, pass `append=True`, so it chains the current and new hook into an instance of the `SequentialHook` class.
 
     </Tip>
 
     Args:
         module (`torch.nn.Module`): The module to attach a hook to.
         hook (`ModelHook`): The hook to attach.
+        append (`bool`, *optional*, defaults to `False`):
+            Whether the hook should be chained with an existing one (if module already contains a hook) or not.
 
     Returns:
         `torch.nn.Module`: The same module, with the hook attached (the module is modified in place, so the result can
         be discarded).
     """
+
+    if append and (getattr(module, "_hf_hook", None) is not None):
+        old_hook = module._hf_hook
+        remove_hook_from_module(module)
+        hook = SequentialHook(old_hook, hook)
+
     if hasattr(module, "_hf_hook") and hasattr(module, "_old_forward"):
         # If we already put some hook on this module, we replace it with the new one.
         old_forward = module._old_forward
@@ -352,7 +360,7 @@ def attach_align_device_hook(
             offload_buffers=offload_buffers,
             place_submodules=full_offload,
         )
-        add_hook_to_module(module, hook)
+        add_hook_to_module(module, hook, append=True)
 
     # We stop the recursion in case we hit the full offload.
     if full_offload:
diff --git a/tests/test_hooks.py b/tests/test_hooks.py
index 9d48db9e1..e3824809c 100644
--- a/tests/test_hooks.py
+++ b/tests/test_hooks.py
@@ -69,6 +69,25 @@ def test_add_and_remove_hooks(self):
         self.assertFalse(hasattr(test_model, "_hf_hook"))
         self.assertFalse(hasattr(test_model, "_old_forward"))
 
+    def test_append_and_remove_hooks(self):
+        test_model = ModelForTest()
+        test_hook = ModelHook()
+
+        add_hook_to_module(test_model, test_hook)
+        add_hook_to_module(test_model, test_hook, append=True)
+
+        self.assertEqual(isinstance(test_model._hf_hook, SequentialHook), True)
+        self.assertEqual(len(test_model._hf_hook.hooks), 2)
+        self.assertTrue(hasattr(test_model, "_old_forward"))
+
+        # Check adding the hook did not change the name or the signature
+        self.assertEqual(test_model.forward.__name__, "forward")
+        self.assertListEqual(list(inspect.signature(test_model.forward).parameters), ["x"])
+
+        remove_hook_from_module(test_model)
+        self.assertFalse(hasattr(test_model, "_hf_hook"))
+        self.assertFalse(hasattr(test_model, "_old_forward"))
+
     def test_pre_forward_hook_is_executed(self):
         test_model = ModelForTest()
         x = torch.randn(2, 3)


Review comments: body: ```suggestion
        append (`bool`, *optional*, defaults to `False`):
            Whether the hook should be chained with an existing one (if module already contains a hook) or not.
```

Also can't comment on line 119 but this should also be mentioned in the Note.
diff_hunk: @@ -123,11 +123,21 @@ def add_hook_to_module(module: nn.Module, hook: ModelHook):
     Args:
         module (`torch.nn.Module`): The module to attach a hook to.
         hook (`ModelHook`): The hook to attach.
+        append (`bool`): Whether, if module already contains a hook, should chain the new one with SequentialHook
body: Those two ifs can be grouped.
Also the second test can be simplified in `getattr(module, "_hf_hook", None) is not None`
diff_hunk: @@ -123,11 +123,21 @@ def add_hook_to_module(module: nn.Module, hook: ModelHook):
     Args:
         module (`torch.nn.Module`): The module to attach a hook to.
         hook (`ModelHook`): The hook to attach.
+        append (`bool`): Whether, if module already contains a hook, should chain the new one with SequentialHook
 
     Returns:
         `torch.nn.Module`: The same module, with the hook attached (the module is modified in place, so the result can
         be discarded).
     """
+
+    if append:
+        if hasattr(module, "_hf_hook") and (module._hf_hook is not None):
body: ```suggestion
            hook = SequentialHook(old_hook, hook)
```
diff_hunk: @@ -123,11 +123,21 @@ def add_hook_to_module(module: nn.Module, hook: ModelHook):
     Args:
         module (`torch.nn.Module`): The module to attach a hook to.
         hook (`ModelHook`): The hook to attach.
+        append (`bool`): Whether, if module already contains a hook, should chain the new one with SequentialHook
 
     Returns:
         `torch.nn.Module`: The same module, with the hook attached (the module is modified in place, so the result can
         be discarded).
     """
+
+    if append:
+        if hasattr(module, "_hf_hook") and (module._hf_hook is not None):
+            old_hook = module._hf_hook
+            remove_hook_from_module(module)
+
+            hooks = [old_hook, hook]
+            hook = SequentialHook(*hooks)
body: Done.
diff_hunk: @@ -123,11 +123,21 @@ def add_hook_to_module(module: nn.Module, hook: ModelHook):
     Args:
         module (`torch.nn.Module`): The module to attach a hook to.
         hook (`ModelHook`): The hook to attach.
+        append (`bool`): Whether, if module already contains a hook, should chain the new one with SequentialHook
body: Done, thanks for the great idea. 
diff_hunk: @@ -123,11 +123,21 @@ def add_hook_to_module(module: nn.Module, hook: ModelHook):
     Args:
         module (`torch.nn.Module`): The module to attach a hook to.
         hook (`ModelHook`): The hook to attach.
+        append (`bool`): Whether, if module already contains a hook, should chain the new one with SequentialHook
 
     Returns:
         `torch.nn.Module`: The same module, with the hook attached (the module is modified in place, so the result can
         be discarded).
     """
+
+    if append:
+        if hasattr(module, "_hf_hook") and (module._hf_hook is not None):
body: Done. Looks much better now.
diff_hunk: @@ -123,11 +123,21 @@ def add_hook_to_module(module: nn.Module, hook: ModelHook):
     Args:
         module (`torch.nn.Module`): The module to attach a hook to.
         hook (`ModelHook`): The hook to attach.
+        append (`bool`): Whether, if module already contains a hook, should chain the new one with SequentialHook
 
     Returns:
         `torch.nn.Module`: The same module, with the hook attached (the module is modified in place, so the result can
         be discarded).
     """
+
+    if append:
+        if hasattr(module, "_hf_hook") and (module._hf_hook is not None):
+            old_hook = module._hf_hook
+            remove_hook_from_module(module)
+
+            hooks = [old_hook, hook]
+            hook = SequentialHook(*hooks)

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: The fix is not exactly right: by doing so, the hook that ensures the input and output of the model are on the same device is now erased. In your code sample in #767, since `x` is on the CPU, `net(x)` should also be on the CPU. This is not the case with your PR. The solution would be to write a util function that will:
- just add the hook if none is present
- extract the current hook if one is present and chain it with this hook using a `SequentialHook`.

This is slightly more advanced than the current PR, so let me know if you'd prefer for me to do it :-)
body: @sgugger I would like to try if that's ok to you. 

What do you think of creating an `append_if_needed` flag on `add_hook_to_module` that does what you just said?
body: That works for me, though the name of the argument could simply be `append` :-)
Thanks for diving into this!
body: @sgugger `append` it is.
body: @sgugger it is ready for review, I've also added the tests. 
body: > Very nice, thanks! Left a couple of nits, and I think you should still put the hook with io first: just tested locally and we still have the same issue of `net(x)` being on the wrong device since it runs second and the input was already moved.

@sgugger I've just addressed your nits and moved the io hook to the top. Thanks for the review! I've tested it locally on the snippet of the bug report and it brings the tensor back to CPU after the inference. Thanks!
body: @sgugger, there is a test step that failed due to an http error when installing a lib. I've created and empty commit to try running it again.