diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 1254c19e9..0fd2a5701 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -22,7 +22,7 @@
 
 from packaging import version
 
-from .checkpointing import load_accelerator_state, save_accelerator_state
+from .checkpointing import load_accelerator_state, load_custom_state, save_accelerator_state, save_custom_state
 from .data_loader import prepare_data_loader
 from .kwargs_handlers import DistributedDataParallelKwargs, GradScalerKwargs, InitProcessGroupKwargs, KwargsHandler
 from .optimizer import AcceleratedOptimizer
@@ -33,6 +33,7 @@
     convert_outputs_to_fp32,
     extract_model_from_parallel,
     gather,
+    get_pretty_name,
     pad_across_processes,
     save,
     wait_for_everyone,
@@ -188,6 +189,7 @@ def __init__(
         # Internal references to the training objects
         self._optimizers = []
         self._models = []
+        self._custom_objects = []
 
         # RNG Types
         self.rng_types = rng_types
@@ -564,7 +566,7 @@ def save(self, obj, f):
 
     def save_state(self, output_dir: str):
         """
-        Saves the current states of the model, optimizer, scaler, and RNG generators.
+        Saves the current states of the model, optimizer, scaler, RNG generators, and registered objects.
 
         Args:
             output_dir (:obj:`str` or :obj:`os.PathLike`):
@@ -575,11 +577,16 @@ def save_state(self, output_dir: str):
         os.makedirs(output_dir, exist_ok=True)
         logger.info(f"Saving current state to {output_dir}")
         weights = [self.get_state_dict(m) for m in self._models]
-        return save_accelerator_state(output_dir, weights, self._optimizers, self.state.process_index, self.scaler)
+        save_location = save_accelerator_state(
+            output_dir, weights, self._optimizers, self.state.process_index, self.scaler
+        )
+        for i, obj in enumerate(self._custom_objects):
+            save_custom_state(obj, output_dir, i)
+        return save_location
 
     def load_state(self, input_dir: str):
         """
-        Loads the current states of the model, optimizer, scaler, and RNG generators.
+        Loads the current states of the model, optimizer, scaler, RNG generators, and registered objects.
 
         Args:
             input_dir (:obj:`str` or :obj:`os.PathLike`):
@@ -591,6 +598,16 @@ def load_state(self, input_dir: str):
             raise ValueError(f"Tried to find {input_dir} but folder does not exist")
         logger.info(f"Loading states from {input_dir}")
         load_accelerator_state(input_dir, self._models, self._optimizers, self.state.process_index, self.scaler)
+        custom_checkpoints = [f for f in os.listdir(input_dir) if "custom_checkpoint" in f]
+        if len(custom_checkpoints) != len(self._custom_objects):
+            err = "Warning! Number of found checkpoints does not match the number of registered objects:"
+            err += f"\n\tFound checkpoints: {len(custom_checkpoints)}"
+            err += f"\n\tRegistered objects: {len(self._custom_objects)}\nSkipping."
+            logger.warn(err)
+        else:
+            logger.info(f"Loading in {len(custom_checkpoints)} custom states")
+            for index, obj in enumerate(self._custom_objects):
+                load_custom_state(obj, input_dir, index)
 
     def free_memory(self):
         """
@@ -646,6 +663,26 @@ def get_state_dict(self, model):
 
         return state_dict
 
+    def register_for_checkpointing(self, *objects):
+        """
+        Makes note of `objects` and will save or load them in during `save_state` or `load_state`.
+
+        These should be utilized when the state is being loaded or saved in the same script. It is not designed to be
+        used in different scripts
+
+        Note: Every `object` must have a `load_state_dict` and `state_dict` function to be stored.
+        """
+        invalid_objects = []
+        for obj in objects:
+            if not hasattr(obj, "state_dict") or not hasattr(obj, "load_state_dict"):
+                invalid_objects.append(obj)
+        if len(invalid_objects) > 0:
+            err = "All `objects` must include a `state_dict` and `load_state_dict` function to be stored. The following inputs are invalid:"
+            for index, obj in enumerate(invalid_objects):
+                err += f"\n\t- Item at index {index}, `{get_pretty_name(obj)}`"
+            raise ValueError(err)
+        self._custom_objects.extend(objects)
+
     @contextmanager
     def autocast(self):
         """
diff --git a/src/accelerate/checkpointing.py b/src/accelerate/checkpointing.py
index 37d68b90c..e1a28d108 100644
--- a/src/accelerate/checkpointing.py
+++ b/src/accelerate/checkpointing.py
@@ -14,6 +14,7 @@
 
 import os
 import random
+from pathlib import Path
 from typing import List
 
 import numpy as np
@@ -21,7 +22,7 @@
 from torch.cuda.amp import GradScaler
 
 from .state import is_tpu_available
-from .utils import MODEL_NAME, OPTIMIZER_NAME, RNG_STATE_NAME, SCALER_NAME, save
+from .utils import MODEL_NAME, OPTIMIZER_NAME, RNG_STATE_NAME, SCALER_NAME, get_pretty_name, save
 
 
 if is_tpu_available():
@@ -132,3 +133,22 @@ def load_accelerator_state(input_dir, models, optimizers, process_index, scaler=
     if is_tpu_available():
         xm.set_rng_state(states["xm_seed"])
     logger.info("All random states loaded successfully")
+
+
+def save_custom_state(obj, path, index: int = 0):
+    """
+    Saves the state of `obj` to `{path}/custom_checkpoint_{index}.pkl`
+    """
+    # Should this be the right way to get a qual_name type value from `obj`?
+    save_location = Path(path) / f"custom_checkpoint_{index}.pkl"
+    logger.info(f"Saving the state of {get_pretty_name(obj)} to {save_location}")
+    torch.save(obj.state_dict(), save_location)
+
+
+def load_custom_state(obj, path, index: int = 0):
+    """
+    Loads the state of `obj` at `{path}/custom_checkpoint_{index}.pkl`
+    """
+    load_location = f"{path}/custom_checkpoint_{index}.pkl"
+    logger.info(f"Loading the state of {get_pretty_name(obj)} from {load_location}")
+    obj.load_state_dict(torch.load(load_location))
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 62b1291ee..4b20cc1c3 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -691,3 +691,16 @@ def patch_environment(**kwargs):
 
     for key in kwargs:
         del os.environ[key.upper()]
+
+
+def get_pretty_name(obj):
+    """
+    Gets a pretty name from ``obj``
+    """
+    if not hasattr(obj, "__qualname__") and not hasattr(obj, "__name__"):
+        obj = getattr(obj, "__class__", obj)
+    if hasattr(obj, "__qualname__"):
+        return obj.__qualname__
+    if hasattr(obj, "__name__"):
+        return obj.__name__
+    return str(obj)
diff --git a/tests/test_state_checkpointing.py b/tests/test_state_checkpointing.py
index a74dcb724..87b2d3b06 100644
--- a/tests/test_state_checkpointing.py
+++ b/tests/test_state_checkpointing.py
@@ -43,13 +43,13 @@ def get_dataset(n_batches):
     return (train_dataloader, valid_dataloader)
 
 
-def train(num_epochs, model, dataloader, optimizer, accelerator):
+def train(num_epochs, model, dataloader, optimizer, accelerator, scheduler=None):
     "Trains for `num_epochs`"
     rands = []
     for epoch in range(num_epochs):
         # Train quickly
         model.train()
-        for step, batch in enumerate(dataloader):
+        for batch in dataloader:
             x, y = batch
             outputs = model(x)
             loss = torch.nn.functional.mse_loss(outputs, y)
@@ -57,6 +57,8 @@ def train(num_epochs, model, dataloader, optimizer, accelerator):
             optimizer.step()
             optimizer.zero_grad()
         rands.append(random.random())  # Introduce some randomness
+        if scheduler is not None:
+            scheduler.step()
     return rands
 
 
@@ -123,3 +125,41 @@ def test_can_resume_training(self):
             self.assertEqual(b1, b3)
             self.assertEqual(opt_state1, opt_state3)
             self.assertEqual(ground_truth_rands, test_rands)
+
+    def test_invalid_registration(self):
+        t = torch.tensor([1, 2, 3])
+        t1 = torch.tensor([2, 3, 4])
+        net = DummyModel()
+        opt = torch.optim.Adam(net.parameters())
+        accelerator = Accelerator()
+        with self.assertRaises(ValueError) as ve:
+            accelerator.register_for_checkpointing(t, t1, net, opt)
+        message = str(ve.exception)
+        self.assertTrue("Item at index 0" in message)
+        self.assertTrue("Item at index 1" in message)
+        self.assertFalse("Item at index 2" in message)
+        self.assertFalse("Item at index 3" in message)
+
+    def test_with_scheduler(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            set_seed(42)
+            model = DummyModel()
+            optimizer = torch.optim.Adam(params=model.parameters(), lr=1e-3)
+            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.99)
+            train_dataloader, valid_dataloader = dummy_dataloaders()
+            # Train baseline
+            accelerator = Accelerator()
+            model, optimizer, train_dataloader, valid_dataloader = accelerator.prepare(
+                model, optimizer, train_dataloader, valid_dataloader
+            )
+            accelerator.register_for_checkpointing(scheduler)
+            # Save initial
+            initial = os.path.join(tmpdir, "initial")
+            accelerator.save_state(initial)
+            scheduler_state = scheduler.state_dict()
+            train(3, model, train_dataloader, optimizer, accelerator, scheduler)
+            self.assertNotEqual(scheduler_state, scheduler.state_dict())
+
+            # Load everything back in and make sure all states work
+            accelerator.load_state(initial)
+            self.assertEqual(scheduler_state, scheduler.state_dict())


Review comments: body: Why the +1 here?
diff_hunk: @@ -591,6 +597,14 @@ def load_state(self, input_dir: str):
             raise ValueError(f"Tried to find {input_dir} but folder does not exist")
         logger.info(f"Loading states from {input_dir}")
         load_accelerator_state(input_dir, self._models, self._optimizers, self.state.process_index, self.scaler)
+        custom_checkpoints = [f for f in os.listdir(input_dir) if "custom_checkpoint" in f]
+        if len(custom_checkpoints) != (len(self._custom_objects) + 1):
body: I would append them to the existing `_custom_objects`.
diff_hunk: @@ -646,6 +660,15 @@ def get_state_dict(self, model):
 
         return state_dict
 
+    def register_for_checkpointing(self, *objects):
+        """
+        Makes note of `objects` and will save or load them in during `save_state` or `load_state`.
+
+        Note: These should be utilized when the state is being loaded or saved in the same script. It is not designed
+        to be used in different scripts
+        """
+        self._custom_objects = list(objects)
body: You should use a Path or `os.path.join` here otherwise it won't work in Windows.
diff_hunk: @@ -132,3 +132,27 @@ def load_accelerator_state(input_dir, models, optimizers, process_index, scaler=
     if is_tpu_available():
         xm.set_rng_state(states["xm_seed"])
     logger.info("All random states loaded successfully")
+
+
+def save_custom_state(obj, path, index: int = 0):
+    """
+    Saves the state of `obj` to `{path}/custom_checkpoint_{index}.pkl`
+    """
+    # Should this be the right way to get a qual_name type value from `obj`?
+    save_location = f"{path}/custom_checkpoint_{index}.pkl"
body: We don't need a torch.load inside here?
diff_hunk: @@ -132,3 +132,27 @@ def load_accelerator_state(input_dir, models, optimizers, process_index, scaler=
     if is_tpu_available():
         xm.set_rng_state(states["xm_seed"])
     logger.info("All random states loaded successfully")
+
+
+def save_custom_state(obj, path, index: int = 0):
+    """
+    Saves the state of `obj` to `{path}/custom_checkpoint_{index}.pkl`
+    """
+    # Should this be the right way to get a qual_name type value from `obj`?
+    save_location = f"{path}/custom_checkpoint_{index}.pkl"
+    logger.info(f"Saving the state of {str(obj)} to {save_location}")
+    if hasattr(obj, "state_dict"):
+        obj = obj.state_dict()
+    torch.save(obj, save_location)
+
+
+def load_custom_state(obj, path, index: int = 0):
+    """
+    Loads the state of `obj` at `{path}/custom_checkpoint_{index}.pkl`
+    """
+    load_location = f"{path}/custom_checkpoint_{index}.pkl"
+    if hasattr(obj, "load_state_dict"):
+        obj = obj.load_state_dict(load_location)
body: For the custom objects + the saved objected was my thought at the time. But you're write its a bug
diff_hunk: @@ -591,6 +597,14 @@ def load_state(self, input_dir: str):
             raise ValueError(f"Tried to find {input_dir} but folder does not exist")
         logger.info(f"Loading states from {input_dir}")
         load_accelerator_state(input_dir, self._models, self._optimizers, self.state.process_index, self.scaler)
+        custom_checkpoints = [f for f in os.listdir(input_dir) if "custom_checkpoint" in f]
+        if len(custom_checkpoints) != (len(self._custom_objects) + 1):
body: The warning message does not match what the test is testing :-)
diff_hunk: @@ -591,6 +598,13 @@ def load_state(self, input_dir: str):
             raise ValueError(f"Tried to find {input_dir} but folder does not exist")
         logger.info(f"Loading states from {input_dir}")
         load_accelerator_state(input_dir, self._models, self._optimizers, self.state.process_index, self.scaler)
+        custom_checkpoints = [f for f in os.listdir(input_dir) if "custom_checkpoint" in f]
+        if len(custom_checkpoints) != len(self._custom_objects):
+            logger.warn("Warning! You are loading a state that does not include registered objects.")
body: This is not a proper docstring **but** it will be once #271 is merged, so giving you a free pass ;-)
diff_hunk: @@ -646,6 +660,26 @@ def get_state_dict(self, model):
 
         return state_dict
 
+    def register_for_checkpointing(self, *objects):
+        """
+        Makes note of `objects` and will save or load them in during `save_state` or `load_state`.
+
+        These should be utilized when the state is being loaded or saved in the same script. It is not designed to be
+        used in different scripts
+
+        Note: Every `object` must have a `load_state_dict` and `state_dict` function to be stored.
body: ```suggestion
    torch.save(obj.state_dict(), save_location)
```
Can be combined now.
diff_hunk: @@ -132,3 +133,23 @@ def load_accelerator_state(input_dir, models, optimizers, process_index, scaler=
     if is_tpu_available():
         xm.set_rng_state(states["xm_seed"])
     logger.info("All random states loaded successfully")
+
+
+def save_custom_state(obj, path, index: int = 0):
+    """
+    Saves the state of `obj` to `{path}/custom_checkpoint_{index}.pkl`
+    """
+    # Should this be the right way to get a qual_name type value from `obj`?
+    save_location = Path(path) / f"custom_checkpoint_{index}.pkl"
+    logger.info(f"Saving the state of {get_pretty_name(obj)} to {save_location}")
+    obj = obj.state_dict()
+    torch.save(obj, save_location)
body: Not sure we need a new class for this. Can be included in the current one!
diff_hunk: @@ -123,3 +125,43 @@ def test_can_resume_training(self):
             self.assertEqual(b1, b3)
             self.assertEqual(opt_state1, opt_state3)
             self.assertEqual(ground_truth_rands, test_rands)
+
+
+class CustomItemsTest(unittest.TestCase):
body: Can you print the two numbers?
diff_hunk: @@ -600,7 +600,9 @@ def load_state(self, input_dir: str):
         load_accelerator_state(input_dir, self._models, self._optimizers, self.state.process_index, self.scaler)
         custom_checkpoints = [f for f in os.listdir(input_dir) if "custom_checkpoint" in f]
         if len(custom_checkpoints) != len(self._custom_objects):
-            logger.warn("Warning! You are loading a state that does not include registered objects.")
+            logger.warn(
+                "Warning! Number of found checkpoints does not match the number of registered objects. Skipping."
body: How does 
```python
            err = "Warning! Number of found checkpoints does not match the number of registered objects:"
            err += f"\n\tFound checkpoints: {len(custom_checkpoints)}"
            err += f"\n\tRegistered objects: {len(self._custom_objects)}\nSkipping."
```
Sound?
diff_hunk: @@ -600,7 +600,9 @@ def load_state(self, input_dir: str):
         load_accelerator_state(input_dir, self._models, self._optimizers, self.state.process_index, self.scaler)
         custom_checkpoints = [f for f in os.listdir(input_dir) if "custom_checkpoint" in f]
         if len(custom_checkpoints) != len(self._custom_objects):
-            logger.warn("Warning! You are loading a state that does not include registered objects.")
+            logger.warn(
+                "Warning! Number of found checkpoints does not match the number of registered objects. Skipping."