diff --git a/examples/by_feature/fsdp_with_peak_mem_tracking.py b/examples/by_feature/fsdp_with_peak_mem_tracking.py
index 8a289442d..8304bcefe 100644
--- a/examples/by_feature/fsdp_with_peak_mem_tracking.py
+++ b/examples/by_feature/fsdp_with_peak_mem_tracking.py
@@ -301,7 +301,7 @@ def collate_fn(examples):
                     {
                         "accuracy": eval_metric["accuracy"],
                         "f1": eval_metric["f1"],
-                        "train_loss": total_loss.item(),
+                        "train_loss": total_loss.item() / len(train_dataloader),
                     },
                     step=epoch,
                 )
diff --git a/examples/by_feature/tracking.py b/examples/by_feature/tracking.py
index 1ccc2c1b0..9538b21b1 100644
--- a/examples/by_feature/tracking.py
+++ b/examples/by_feature/tracking.py
@@ -222,7 +222,7 @@ def training_function(config, args):
                 {
                     "accuracy": eval_metric["accuracy"],
                     "f1": eval_metric["f1"],
-                    "train_loss": total_loss.item(),
+                    "train_loss": total_loss.item() / len(train_dataloader),
                     "epoch": epoch,
                 },
                 step=epoch,
diff --git a/examples/complete_cv_example.py b/examples/complete_cv_example.py
index fd830442d..bd2ed3b29 100644
--- a/examples/complete_cv_example.py
+++ b/examples/complete_cv_example.py
@@ -258,7 +258,12 @@ def training_function(config, args):
         accelerator.print(f"epoch {epoch}: {100 * eval_metric:.2f}")
         if args.with_tracking:
             accelerator.log(
-                {"accuracy": 100 * eval_metric, "total_loss": total_loss, "epoch": epoch}, step=overall_step
+                {
+                    "accuracy": 100 * eval_metric,
+                    "train_loss": total_loss.item() / len(train_dataloader),
+                    "epoch": epoch,
+                },
+                step=overall_step,
             )
         if checkpointing_steps == "epoch":
             output_dir = f"epoch_{epoch}"
diff --git a/examples/complete_nlp_example.py b/examples/complete_nlp_example.py
index abf85c8d9..9c8b89706 100644
--- a/examples/complete_nlp_example.py
+++ b/examples/complete_nlp_example.py
@@ -246,7 +246,7 @@ def collate_fn(examples):
                 {
                     "accuracy": eval_metric["accuracy"],
                     "f1": eval_metric["f1"],
-                    "train_loss": total_loss.item(),
+                    "train_loss": total_loss.item() / len(train_dataloader),
                     "epoch": epoch,
                 },
                 step=epoch,
diff --git a/tests/test_examples.py b/tests/test_examples.py
index af892f4eb..6ec2a143a 100644
--- a/tests/test_examples.py
+++ b/tests/test_examples.py
@@ -100,13 +100,13 @@ def test_cv_examples(self):
         cv_path = os.path.abspath(os.path.join("examples", "cv_example.py"))
         special_strings = [
             " " * 16 + "{\n\n",
-            " " * 18 + '"accuracy": eval_metric["accuracy"],\n\n',
-            " " * 18 + '"f1": eval_metric["f1"],\n\n',
-            " " * 18 + '"train_loss": total_loss.item(),\n\n',
-            " " * 18 + '"epoch": epoch,\n\n',
+            " " * 20 + '"accuracy": eval_metric["accuracy"],\n\n',
+            " " * 20 + '"f1": eval_metric["f1"],\n\n',
+            " " * 20 + '"train_loss": total_loss.item() / len(train_dataloader),\n\n',
+            " " * 20 + '"epoch": epoch,\n\n',
             " " * 16 + "},\n\n",
             " " * 16 + "step=epoch,\n",
-            " " * 8,
+            " " * 12,
         ]
         self.one_complete_example("complete_cv_example.py", True, cv_path, special_strings)
         self.one_complete_example("complete_cv_example.py", False, cv_path, special_strings)



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: @pacman100 @sgugger In a multi-gpu environment, does the **total_loss** here represent the loss of all training data in the entire epoch or just the loss of training data on the main process? 

I tried to print the result of total_loss on each process, and it seems that accelerator.log() only records the total_loss on the main process. So, please, do I need to use accelerator.gather() to get the training loss on the whole epoch? I hope I can get your confirmation, many thanks!
body: Hello @benjpau , yes, you are correct, you would need to `gather` to get the total epoch loss. Will raise a PR to fix the examples to reflect the same.