diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 81759962c..287e6899a 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -647,7 +647,15 @@ def _prepare_fsdp(self, *args):
                         "FSDP Warning: When using FSDP, several parameter groups will be conflated into "
                         "a single one due to nested module wrapping and parameter flattening."
                     )
-                optimizer = obj.optimizer.__class__(model.parameters(), **obj.optimizer.defaults)
+                try:
+                    optimizer = obj.optimizer.__class__(model.parameters(), **obj.optimizer.defaults)
+                except TypeError:
+                    if "differentiable" in obj.optimizer.defaults:
+                        # https://github.com/huggingface/accelerate/issues/801
+                        defaults = {k: v for k, v in obj.optimizer.defaults.items() if k != "differentiable"}
+                        optimizer = obj.optimizer.__class__(model.parameters(), **defaults)
+                    else:
+                        raise
                 obj = self.prepare_optimizer(optimizer)
                 optimizers.append(obj)
             elif isinstance(obj, torch.nn.Module):


Review comments: body: This will delete inplace, we should probably create a new dict here
diff_hunk: @@ -647,7 +647,15 @@ def _prepare_fsdp(self, *args):
                         "FSDP Warning: When using FSDP, several parameter groups will be conflated into "
                         "a single one due to nested module wrapping and parameter flattening."
                     )
-                optimizer = obj.optimizer.__class__(model.parameters(), **obj.optimizer.defaults)
+                try:
+                    optimizer = obj.optimizer.__class__(model.parameters(), **obj.optimizer.defaults)
+                except TypeError:
+                    if "differentiable" in obj.optimizer.defaults:
+                        # https://github.com/huggingface/accelerate/issues/801
+                        del obj.optimizer.defaults["differentiable"]
body: ```suggestion
                        defaults = obj.optimizer.defaults.copy()
                        del defaults["differentiable"]
                        optimizer = obj.optimizer.__class__(model.parameters(), **defaults)
```
diff_hunk: @@ -647,7 +647,15 @@ def _prepare_fsdp(self, *args):
                         "FSDP Warning: When using FSDP, several parameter groups will be conflated into "
                         "a single one due to nested module wrapping and parameter flattening."
                     )
-                optimizer = obj.optimizer.__class__(model.parameters(), **obj.optimizer.defaults)
+                try:
+                    optimizer = obj.optimizer.__class__(model.parameters(), **obj.optimizer.defaults)
+                except TypeError:
+                    if "differentiable" in obj.optimizer.defaults:
+                        # https://github.com/huggingface/accelerate/issues/801
+                        del obj.optimizer.defaults["differentiable"]
+                        optimizer = obj.optimizer.__class__(model.parameters(), **obj.optimizer.defaults)
body: Good point. Done. PTAL.
diff_hunk: @@ -647,7 +647,15 @@ def _prepare_fsdp(self, *args):
                         "FSDP Warning: When using FSDP, several parameter groups will be conflated into "
                         "a single one due to nested module wrapping and parameter flattening."
                     )
-                optimizer = obj.optimizer.__class__(model.parameters(), **obj.optimizer.defaults)
+                try:
+                    optimizer = obj.optimizer.__class__(model.parameters(), **obj.optimizer.defaults)
+                except TypeError:
+                    if "differentiable" in obj.optimizer.defaults:
+                        # https://github.com/huggingface/accelerate/issues/801
+                        del obj.optimizer.defaults["differentiable"]
body: ```suggestion
                        defaults = {k: v for k, v in obj.optimizer.defaults.items() if k != "differentiable"}
```
diff_hunk: @@ -647,7 +647,16 @@ def _prepare_fsdp(self, *args):
                         "FSDP Warning: When using FSDP, several parameter groups will be conflated into "
                         "a single one due to nested module wrapping and parameter flattening."
                     )
-                optimizer = obj.optimizer.__class__(model.parameters(), **obj.optimizer.defaults)
+                try:
+                    optimizer = obj.optimizer.__class__(model.parameters(), **obj.optimizer.defaults)
+                except TypeError:
+                    if "differentiable" in obj.optimizer.defaults:
+                        # https://github.com/huggingface/accelerate/issues/801
+                        defaults = obj.optimizer.defaults.copy()
+                        del defaults["differentiable"]

Discussion: body: closing #801
body: _The documentation is not available anymore as the PR was closed or merged._
body: Thank you for the quick fix! 🤗