diff --git a/src/accelerate/state.py b/src/accelerate/state.py
index e341ad93f..06e9cb8fa 100644
--- a/src/accelerate/state.py
+++ b/src/accelerate/state.py
@@ -75,6 +75,7 @@ def __init__(
         self.__dict__ = self._shared_state
         if parse_flag_from_env("USE_CPU"):
             cpu = True
+        self._check_initialized(mixed_precision, cpu)
         self.fork_launched = parse_flag_from_env("FORK_LAUNCHED", 0)
         if not getattr(self, "initialized", False):
             self.backend = None
@@ -196,3 +197,17 @@ def __repr__(self):
     @property
     def use_fp16(self):
         return self.mixed_precision != "no"
+
+    @staticmethod
+    def _reset_state():
+        "Resets `_shared_state`, is used internally and should not be called"
+        AcceleratorState._shared_state = {}
+
+    def _check_initialized(self, mixed_precision=None, cpu=None):
+        "Checks if a modification is trying to be made and the `AcceleratorState` has already been initialized"
+        if getattr(self, "initialized", False):
+            err = "AcceleratorState has already been initialized and cannot be changed, restart your runtime completely and pass `{flag}` to `Accelerate()`."
+            if cpu and self.device.type != "cpu":
+                raise ValueError(err.format(flag="cpu=True"))
+            if mixed_precision is not None and mixed_precision != self.mixed_precision:
+                raise ValueError(err.format(flag=f"mixed_precision='{mixed_precision}'"))
diff --git a/src/accelerate/test_utils/scripts/test_script.py b/src/accelerate/test_utils/scripts/test_script.py
index 8ed869edc..018564fa0 100644
--- a/src/accelerate/test_utils/scripts/test_script.py
+++ b/src/accelerate/test_utils/scripts/test_script.py
@@ -247,6 +247,7 @@ def training_check():
 
     # Mostly a test that FP16 doesn't crash as the operation inside the model is not converted to FP16
     print("FP16 training check.")
+    AcceleratorState._reset_state()
     accelerator = Accelerator(mixed_precision="fp16")
     train_dl = DataLoader(train_set, batch_size=batch_size, shuffle=True, generator=generator)
     model = RegressionModel()
@@ -269,6 +270,7 @@ def training_check():
 
     # TEST that previous fp16 flag still works
     print("Legacy FP16 training check.")
+    AcceleratorState._reset_state()
     accelerator = Accelerator(fp16=True)
     train_dl = DataLoader(train_set, batch_size=batch_size, shuffle=True, generator=generator)
     model = RegressionModel()
@@ -291,6 +293,7 @@ def training_check():
 
     # Mostly a test that BF16 doesn't crash as the operation inside the model is not converted to BF16
     print("BF16 training check.")
+    AcceleratorState._reset_state()
     accelerator = Accelerator(mixed_precision="bf16")
     train_dl = DataLoader(train_set, batch_size=batch_size, shuffle=True, generator=generator)
     model = RegressionModel()


Review comments: body: I don't understand why we are using this class, when you can just do a regular string with flag between curly brackets and then use the format method.
diff_hunk: @@ -196,3 +198,14 @@ def __repr__(self):
     @property
     def use_fp16(self):
         return self.mixed_precision != "no"
+
+    def _check_initialized(self, mixed_precision=None, cpu=None):
+        "Checks if a modification is trying to be made and the `AcceleratorState` has already been initialized"
+        if getattr(self, "initialized", False):
+            template = Template(
+                "AcceleratorState has already been initialized and cannot be changed, restart your runtime completely and pass `$flag` to `Accelerate()`."
+            )
body: Definitely not because I forgot strings could do this... 👀 
diff_hunk: @@ -196,3 +198,14 @@ def __repr__(self):
     @property
     def use_fp16(self):
         return self.mixed_precision != "no"
+
+    def _check_initialized(self, mixed_precision=None, cpu=None):
+        "Checks if a modification is trying to be made and the `AcceleratorState` has already been initialized"
+        if getattr(self, "initialized", False):
+            template = Template(
+                "AcceleratorState has already been initialized and cannot be changed, restart your runtime completely and pass `$flag` to `Accelerate()`."
+            )
body: Should be a `staticmethod` no?
diff_hunk: @@ -196,3 +197,17 @@ def __repr__(self):
     @property
     def use_fp16(self):
         return self.mixed_precision != "no"
+
+    @classmethod
body: And here you should change the value of `AcceleratorState._shared_state`
diff_hunk: @@ -196,3 +197,17 @@ def __repr__(self):
     @property
     def use_fp16(self):
         return self.mixed_precision != "no"
+
+    @classmethod
+    def _reset_state(cls):
+        "Resets `_shared_state`, is used internally and should not be called"
+        cls._shared_state = {}

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._