diff --git a/examples/by_feature/checkpointing.py b/examples/by_feature/checkpointing.py
index e595a8db7..ee0716b6d 100644
--- a/examples/by_feature/checkpointing.py
+++ b/examples/by_feature/checkpointing.py
@@ -166,6 +166,8 @@ def training_function(config, args):
     # New Code #
     # We need to keep track of how many total steps we have iterated over
     overall_step = 0
+    # We also need to keep track of the stating epoch so files are named properly
+    starting_epoch = 0
 
     # We need to load the checkpoint back in before training here with `load_state`
     # The total number of epochs is adjusted based on where the state is being loaded from,
@@ -184,23 +186,23 @@ def training_function(config, args):
         training_difference = os.path.splitext(path)[0]
 
         if "epoch" in training_difference:
-            num_epochs -= int(training_difference.replace("epoch_", ""))
+            starting_epoch = int(training_difference.replace("epoch_", "")) + 1
             resume_step = None
         else:
             resume_step = int(training_difference.replace("step_", ""))
-            num_epochs -= resume_step // len(train_dataloader)
-            # If resuming by step, we also need to know exactly how far into the DataLoader we went
-            resume_step = (num_epochs * len(train_dataloader)) - resume_step
+            starting_epoch = resume_step // len(train_dataloader)
+            resume_step -= starting_epoch * len(train_dataloader)
 
     # Now we train the model
-    for epoch in range(num_epochs):
+    for epoch in range(starting_epoch, num_epochs):
         model.train()
         for step, batch in enumerate(train_dataloader):
             # New Code #
             # We need to skip steps until we reach the resumed step during the first epoch
-            if args.resume_from_checkpoint and epoch == 0:
+            if args.resume_from_checkpoint and epoch == starting_epoch:
                 if resume_step is not None and step < resume_step:
-                    pass
+                    overall_step += 1
+                    continue
             # We could avoid this line since we set the accelerator with `device_placement=True`.
             batch.to(accelerator.device)
             outputs = model(**batch)
@@ -246,7 +248,7 @@ def training_function(config, args):
 
         # New Code #
         # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
-        # These are saved to folders named `step_{overall_step}`
+        # These are saved to folders named `epoch_{epoch}`
         # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
         # If mixed precision was used, will also save a "scalar.bin" file
         if checkpointing_steps == "epoch":
diff --git a/examples/complete_cv_example.py b/examples/complete_cv_example.py
index 01033498c..80a083a8d 100644
--- a/examples/complete_cv_example.py
+++ b/examples/complete_cv_example.py
@@ -173,6 +173,10 @@ def training_function(config, args):
     model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare(
         model, optimizer, train_dataloader, eval_dataloader, lr_scheduler
     )
+    # We need to keep track of how many total steps we have iterated over
+    overall_step = 0
+    # We also need to keep track of the stating epoch so files are named properly
+    starting_epoch = 0
 
     # Potentially load in the weights and states from a previous save
     if args.resume_from_checkpoint:
@@ -189,25 +193,24 @@ def training_function(config, args):
         training_difference = os.path.splitext(path)[0]
 
         if "epoch" in training_difference:
-            num_epochs -= int(training_difference.replace("epoch_", ""))
+            starting_epoch = int(training_difference.replace("epoch_", "")) + 1
             resume_step = None
         else:
             resume_step = int(training_difference.replace("step_", ""))
-            num_epochs -= resume_step // len(train_dataloader)
-            # If resuming by step, we also need to know exactly how far into the DataLoader we went
-            resume_step = (num_epochs * len(train_dataloader)) - resume_step
+            starting_epoch = resume_step // len(train_dataloader)
+            resume_step -= starting_epoch * len(train_dataloader)
 
-    overall_step = 0
     # Now we train the model
-    for epoch in range(num_epochs):
+    for epoch in range(starting_epoch, num_epochs):
         model.train()
         if args.with_tracking:
             total_loss = 0
         for step, batch in enumerate(train_dataloader):
             # We need to skip steps until we reach the resumed step
-            if args.resume_from_checkpoint and epoch == 0:
+            if args.resume_from_checkpoint and epoch == starting_epoch:
                 if resume_step is not None and step < resume_step:
-                    pass
+                    overall_step += 1
+                    continue
             # We could avoid this line since we set the accelerator with `device_placement=True`.
             batch = {k: v.to(accelerator.device) for k, v in batch.items()}
             inputs = (batch["image"] - mean) / std
diff --git a/examples/complete_nlp_example.py b/examples/complete_nlp_example.py
index a0e8d568c..1910a4f83 100644
--- a/examples/complete_nlp_example.py
+++ b/examples/complete_nlp_example.py
@@ -153,7 +153,10 @@ def collate_fn(examples):
         model, optimizer, train_dataloader, eval_dataloader, lr_scheduler
     )
 
+    # We need to keep track of how many total steps we have iterated over
     overall_step = 0
+    # We also need to keep track of the stating epoch so files are named properly
+    starting_epoch = 0
 
     # Potentially load in the weights and states from a previous save
     if args.resume_from_checkpoint:
@@ -170,24 +173,24 @@ def collate_fn(examples):
         training_difference = os.path.splitext(path)[0]
 
         if "epoch" in training_difference:
-            num_epochs -= int(training_difference.replace("epoch_", ""))
+            starting_epoch = int(training_difference.replace("epoch_", "")) + 1
             resume_step = None
         else:
             resume_step = int(training_difference.replace("step_", ""))
-            num_epochs -= resume_step // len(train_dataloader)
-            # If resuming by step, we also need to know exactly how far into the DataLoader we went
-            resume_step = (num_epochs * len(train_dataloader)) - resume_step
+            starting_epoch = resume_step // len(train_dataloader)
+            resume_step -= starting_epoch * len(train_dataloader)
 
     # Now we train the model
-    for epoch in range(num_epochs):
+    for epoch in range(starting_epoch, num_epochs):
         model.train()
         if args.with_tracking:
             total_loss = 0
         for step, batch in enumerate(train_dataloader):
             # We need to skip steps until we reach the resumed step
-            if args.resume_from_checkpoint and epoch == 0:
+            if args.resume_from_checkpoint and epoch == starting_epoch:
                 if resume_step is not None and step < resume_step:
-                    pass
+                    overall_step += 1
+                    continue
             # We could avoid this line since we set the accelerator with `device_placement=True`.
             batch.to(accelerator.device)
             outputs = model(**batch)
diff --git a/src/accelerate/test_utils/testing.py b/src/accelerate/test_utils/testing.py
index e348e161f..ac6df7741 100644
--- a/src/accelerate/test_utils/testing.py
+++ b/src/accelerate/test_utils/testing.py
@@ -69,6 +69,8 @@ class TempDirTestCase(unittest.TestCase):
     The temporary directory location will be stored in `self.tmpdir`
     """
 
+    clear_on_setup = True
+
     @classmethod
     def setUpClass(cls):
         "Creates a `tempfile.TemporaryDirectory` and stores it in `cls.tmpdir`"
@@ -82,11 +84,12 @@ def tearDownClass(cls):
 
     def setUp(self):
         "Destroy all contents in `self.tmpdir`, but not `self.tmpdir`"
-        for path in Path(self.tmpdir).glob("**/*"):
-            if path.is_file():
-                path.unlink()
-            elif path.is_dir():
-                shutil.rmtree(path)
+        if self.clear_on_setup:
+            for path in Path(self.tmpdir).glob("**/*"):
+                if path.is_file():
+                    path.unlink()
+                elif path.is_dir():
+                    shutil.rmtree(path)
 
 
 class MockingTestCase(unittest.TestCase):
diff --git a/tests/test_examples.py b/tests/test_examples.py
index 3ecd3bf91..aa78fed9f 100644
--- a/tests/test_examples.py
+++ b/tests/test_examples.py
@@ -22,7 +22,7 @@
 
 from accelerate import DistributedType
 from accelerate.test_utils.examples import compare_against_test
-from accelerate.test_utils.testing import slow
+from accelerate.test_utils.testing import TempDirTestCase, slow
 from datasets import load_dataset
 from transformers import AutoTokenizer
 
@@ -158,30 +158,61 @@ def test_cv_examples(self):
         self.one_complete_example("complete_cv_example.py", False, cv_path, special_strings)
 
 
-class FeatureExamplesTests(unittest.TestCase):
+class FeatureExamplesTests(TempDirTestCase):
+    clear_on_setup = False
+
     @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
     def test_checkpointing_by_epoch(self):
-        with tempfile.TemporaryDirectory() as tmpdir:
-            testargs = f"""
-            checkpointing.py
-            --checkpointing_steps epoch
-            --output_dir {tmpdir}
-            """.split()
+        testargs = f"""
+        checkpointing.py
+        --checkpointing_steps epoch
+        --output_dir {self.tmpdir}
+        """.split()
+        with mock.patch.object(sys, "argv", testargs):
+            checkpointing.main()
+            self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "epoch_1")))
+
+    @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
+    def test_checkpointing_by_steps(self):
+        testargs = f"""
+        checkpointing.py
+        --checkpointing_steps 2
+        --output_dir {self.tmpdir}
+        """.split()
+        with mock.patch.object(sys, "argv", testargs):
+            checkpointing.main()
+            self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "step_4")))
+
+    @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
+    def test_load_states_by_epoch(self):
+        testargs = f"""
+        checkpointing.py
+        --resume_from_checkpoint {os.path.join(self.tmpdir, "epoch_1")}
+        """.split()
+        dummy_results = {"accuracy": mock.ANY, "f1": mock.ANY}
+        with mock.patch("accelerate.Accelerator.print") as mocked_print:
             with mock.patch.object(sys, "argv", testargs):
                 checkpointing.main()
-                self.assertTrue(os.path.exists(os.path.join(tmpdir, "epoch_0")))
+            with self.assertRaises(AssertionError):
+                mocked_print.assert_any_call("epoch 0:", dummy_results)
+            with self.assertRaises(AssertionError):
+                mocked_print.assert_any_call("epoch 1:", dummy_results)
+            mocked_print.assert_any_call("epoch 2:", dummy_results)
 
     @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
-    def test_checkpointing_by_steps(self):
-        with tempfile.TemporaryDirectory() as tmpdir:
-            testargs = f"""
-            checkpointing.py
-            --checkpointing_steps 2
-            --output_dir {tmpdir}
-            """.split()
+    def test_load_states_by_steps(self):
+        testargs = f"""
+        checkpointing.py
+        --resume_from_checkpoint {os.path.join(self.tmpdir, "step_4")}
+        """.split()
+        dummy_results = {"accuracy": mock.ANY, "f1": mock.ANY}
+        with mock.patch("accelerate.Accelerator.print") as mocked_print:
             with mock.patch.object(sys, "argv", testargs):
                 checkpointing.main()
-                self.assertTrue(os.path.exists(os.path.join(tmpdir, "step_2")))
+            with self.assertRaises(AssertionError):
+                mocked_print.assert_any_call("epoch 0:", dummy_results)
+            mocked_print.assert_any_call("epoch 1:", dummy_results)
+            mocked_print.assert_any_call("epoch 2:", dummy_results)
 
     @slow
     def test_cross_validation(self):


Review comments: body: How are we sure we will get those values exactly?
diff_hunk: @@ -158,30 +158,59 @@ def test_cv_examples(self):
         self.one_complete_example("complete_cv_example.py", False, cv_path, special_strings)
 
 
-class FeatureExamplesTests(unittest.TestCase):
+class FeatureExamplesTests(TempDirTestCase):
+    clear_on_setup = False
+
     @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
     def test_checkpointing_by_epoch(self):
-        with tempfile.TemporaryDirectory() as tmpdir:
-            testargs = f"""
-            checkpointing.py
-            --checkpointing_steps epoch
-            --output_dir {tmpdir}
-            """.split()
+        testargs = f"""
+        checkpointing.py
+        --checkpointing_steps epoch
+        --output_dir {self.tmpdir}
+        """.split()
+        with mock.patch.object(sys, "argv", testargs):
+            checkpointing.main()
+            self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "epoch_1")))
+
+    @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
+    def test_checkpointing_by_steps(self):
+        testargs = f"""
+        checkpointing.py
+        --checkpointing_steps 2
+        --output_dir {self.tmpdir}
+        """.split()
+        with mock.patch.object(sys, "argv", testargs):
+            checkpointing.main()
+            self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "step_4")))
+
+    @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
+    def test_load_states_by_epoch(self):
+        testargs = f"""
+        checkpointing.py
+        --resume_from_checkpoint {os.path.join(self.tmpdir, "epoch_1")}
+        """.split()
+        with mock.patch("accelerate.Accelerator.print") as mocked_print:
             with mock.patch.object(sys, "argv", testargs):
                 checkpointing.main()
-                self.assertTrue(os.path.exists(os.path.join(tmpdir, "epoch_0")))
+            with self.assertRaises(AssertionError):
+                mocked_print.assert_any_call("epoch 0:", {"accuracy": 0.5, "f1": 0.0})
body: This stems from the scheduler we use in the examples, it makes it impossible for the model to train quickly so we always get an accuracy of .5 and an f1 of 0 for all of our epochs. Hence why none of these example tests check for if we get "good" accuracy, it's for independent behavior.

*But* I dug deeper and found `mock.ANY`. In this case we only care about matching the `epoch *` text, not the results. So instead we have something like this:

```python
dummy_results = {"accuracy":mock.ANY, "f1":mock.ANY}
with self.assertRaises(AssertionError):
    mocked_print.assert_any_call("epoch 0:", dummy_results)
```

Which helps me sleep much better at night
diff_hunk: @@ -158,30 +158,59 @@ def test_cv_examples(self):
         self.one_complete_example("complete_cv_example.py", False, cv_path, special_strings)
 
 
-class FeatureExamplesTests(unittest.TestCase):
+class FeatureExamplesTests(TempDirTestCase):
+    clear_on_setup = False
+
     @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
     def test_checkpointing_by_epoch(self):
-        with tempfile.TemporaryDirectory() as tmpdir:
-            testargs = f"""
-            checkpointing.py
-            --checkpointing_steps epoch
-            --output_dir {tmpdir}
-            """.split()
+        testargs = f"""
+        checkpointing.py
+        --checkpointing_steps epoch
+        --output_dir {self.tmpdir}
+        """.split()
+        with mock.patch.object(sys, "argv", testargs):
+            checkpointing.main()
+            self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "epoch_1")))
+
+    @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
+    def test_checkpointing_by_steps(self):
+        testargs = f"""
+        checkpointing.py
+        --checkpointing_steps 2
+        --output_dir {self.tmpdir}
+        """.split()
+        with mock.patch.object(sys, "argv", testargs):
+            checkpointing.main()
+            self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "step_4")))
+
+    @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
+    def test_load_states_by_epoch(self):
+        testargs = f"""
+        checkpointing.py
+        --resume_from_checkpoint {os.path.join(self.tmpdir, "epoch_1")}
+        """.split()
+        with mock.patch("accelerate.Accelerator.print") as mocked_print:
             with mock.patch.object(sys, "argv", testargs):
                 checkpointing.main()
-                self.assertTrue(os.path.exists(os.path.join(tmpdir, "epoch_0")))
+            with self.assertRaises(AssertionError):
+                mocked_print.assert_any_call("epoch 0:", {"accuracy": 0.5, "f1": 0.0})
body: You never know if your model might have learned a tiny something, so yes, much better :-)
diff_hunk: @@ -158,30 +158,59 @@ def test_cv_examples(self):
         self.one_complete_example("complete_cv_example.py", False, cv_path, special_strings)
 
 
-class FeatureExamplesTests(unittest.TestCase):
+class FeatureExamplesTests(TempDirTestCase):
+    clear_on_setup = False
+
     @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
     def test_checkpointing_by_epoch(self):
-        with tempfile.TemporaryDirectory() as tmpdir:
-            testargs = f"""
-            checkpointing.py
-            --checkpointing_steps epoch
-            --output_dir {tmpdir}
-            """.split()
+        testargs = f"""
+        checkpointing.py
+        --checkpointing_steps epoch
+        --output_dir {self.tmpdir}
+        """.split()
+        with mock.patch.object(sys, "argv", testargs):
+            checkpointing.main()
+            self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "epoch_1")))
+
+    @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
+    def test_checkpointing_by_steps(self):
+        testargs = f"""
+        checkpointing.py
+        --checkpointing_steps 2
+        --output_dir {self.tmpdir}
+        """.split()
+        with mock.patch.object(sys, "argv", testargs):
+            checkpointing.main()
+            self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "step_4")))
+
+    @mock.patch("checkpointing.get_dataloaders", mocked_dataloaders)
+    def test_load_states_by_epoch(self):
+        testargs = f"""
+        checkpointing.py
+        --resume_from_checkpoint {os.path.join(self.tmpdir, "epoch_1")}
+        """.split()
+        with mock.patch("accelerate.Accelerator.print") as mocked_print:
             with mock.patch.object(sys, "argv", testargs):
                 checkpointing.main()
-                self.assertTrue(os.path.exists(os.path.join(tmpdir, "epoch_0")))
+            with self.assertRaises(AssertionError):
+                mocked_print.assert_any_call("epoch 0:", {"accuracy": 0.5, "f1": 0.0})

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: @sgugger 50/50 on whether to consider these "slow" tests or not. They add ~1.5 min to the CI