diff --git a/src/accelerate/utils/dataclasses.py b/src/accelerate/utils/dataclasses.py
index dc8883425..4f9617711 100644
--- a/src/accelerate/utils/dataclasses.py
+++ b/src/accelerate/utils/dataclasses.py
@@ -23,6 +23,7 @@
 import os
 import typing
 import warnings
+from contextlib import contextmanager
 from dataclasses import dataclass, field
 from datetime import timedelta
 from distutils.util import strtobool
@@ -529,6 +530,23 @@ def set_deepspeed_weakref(self):
 
             self.dschf = HfDeepSpeedConfig(ds_config)  # keep this object alive # noqa
 
+    def is_zero3_init_enabled(self):
+        return self.zero3_init_flag
+
+    @contextmanager
+    def zero3_init_context_manager(self, enable=False):
+        old = self.zero3_init_flag
+        if old == enable:
+            yield
+        else:
+            self.zero3_init_flag = enable
+            self.dschf = None
+            self.set_deepspeed_weakref()
+            yield
+            self.zero3_init_flag = old
+            self.dschf = None
+            self.set_deepspeed_weakref()
+
 
 @dataclass
 class FullyShardedDataParallelPlugin:


Review comments: body: Might be clearer to have an `else` block here instead of the early `return`.
diff_hunk: @@ -529,6 +530,23 @@ def set_deepspeed_weakref(self):
 
             self.dschf = HfDeepSpeedConfig(ds_config)  # keep this object alive # noqa
 
+    def is_zero3_init_enabled(self):
+        return self.zero3_init_flag
+
+    @contextmanager
+    def set_zero3_init(self, enable=False):
+        old = self.zero3_init_flag
+        if old == enable:
+            yield
+            return
+        self.zero3_init_flag = enable
+        self.dschf = None
+        self.set_deepspeed_weakref()
+        yield
+        self.zero3_init_flag = old
+        self.dschf = None
+        self.set_deepspeed_weakref()
body: Done. 
diff_hunk: @@ -529,6 +530,23 @@ def set_deepspeed_weakref(self):
 
             self.dschf = HfDeepSpeedConfig(ds_config)  # keep this object alive # noqa
 
+    def is_zero3_init_enabled(self):
+        return self.zero3_init_flag
+
+    @contextmanager
+    def set_zero3_init(self, enable=False):
+        old = self.zero3_init_flag
+        if old == enable:
+            yield
+            return
+        self.zero3_init_flag = enable
+        self.dschf = None
+        self.set_deepspeed_weakref()
+        yield
+        self.zero3_init_flag = old
+        self.dschf = None
+        self.set_deepspeed_weakref()

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: here is how I used your new method:

```
        from transformer.utils import ContextManagers
        deepspeed_plugin = get_deepspeed_plugin()
        if deepspeed_plugin is not None:
            zero_init_disabled_context = [deepspeed_plugin.set_zero3_init(enable=False)]
        else:
            zero_init_disabled_context = []

        with ContextManagers(zero_init_disabled_context):
            config = AutoConfig.from_pretrained(vision_model_name, **vision_model_params)
            model = AutoModel.from_config(config, torch_dtype=torch_dtype)
```
body: > Would it be more intuitive to call it `zero_init_context_manager`?

renamed it to `zero3_init_context_manager`