diff --git a/examples/by_feature/fsdp_with_peak_mem_tracking.py b/examples/by_feature/fsdp_with_peak_mem_tracking.py
index a8546c3c8..581c5fea0 100644
--- a/examples/by_feature/fsdp_with_peak_mem_tracking.py
+++ b/examples/by_feature/fsdp_with_peak_mem_tracking.py
@@ -103,11 +103,12 @@ def training_function(config, args):
 
     # We need to initialize the trackers we use, and also store our configuration
     if args.with_tracking:
-        run = os.path.split(__file__)[-1].split(".")[0]
-        if args.logging_dir:
-            run = os.path.join(args.logging_dir, run)
-            accelerator.print(run)
-        accelerator.init_trackers(run, config)
+        if accelerator.is_main_process:
+            run = os.path.split(__file__)[-1].split(".")[0]
+            if args.logging_dir:
+                run = os.path.join(args.logging_dir, run)
+                accelerator.print(run)
+            accelerator.init_trackers(run, config)
 
     tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
     datasets = load_dataset("glue", "mrpc")
@@ -293,7 +294,7 @@ def collate_fn(examples):
                     {
                         "accuracy": eval_metric["accuracy"],
                         "f1": eval_metric["f1"],
-                        "train_loss": total_loss,
+                        "train_loss": total_loss.item(),
                     },
                     step=epoch,
                 )
diff --git a/examples/by_feature/tracking.py b/examples/by_feature/tracking.py
index 19f8e9dea..47e15c0cf 100644
--- a/examples/by_feature/tracking.py
+++ b/examples/by_feature/tracking.py
@@ -161,10 +161,11 @@ def training_function(config, args):
     # New Code #
     # We need to initalize the trackers we use. Overall configurations can also be stored
     if args.with_tracking:
-        run = os.path.split(__file__)[-1].split(".")[0]
-        if args.logging_dir:
-            run = os.path.join(args.logging_dir, run)
-        accelerator.init_trackers(run, config)
+        if accelerator.is_main_process:
+            run = os.path.split(__file__)[-1].split(".")[0]
+            if args.logging_dir:
+                run = os.path.join(args.logging_dir, run)
+            accelerator.init_trackers(run, config)
 
     # Now we train the model
     for epoch in range(num_epochs):
@@ -208,15 +209,16 @@ def training_function(config, args):
 
         # New Code #
         # To actually log, we call `Accelerator.log`
-        # The values passed can be of `str`, `int`, or `float`
+        # The values passed can be of `str`, `int`, `float` or `dict` of `str` to `float`/`int`
         if args.with_tracking:
             accelerator.log(
                 {
                     "accuracy": eval_metric["accuracy"],
                     "f1": eval_metric["f1"],
-                    "train_loss": total_loss,
+                    "train_loss": total_loss.item(),
                     "epoch": epoch,
-                }
+                },
+                step=epoch,
             )
 
     # New Code #
diff --git a/examples/complete_cv_example.py b/examples/complete_cv_example.py
index 79e9289b8..fd830442d 100644
--- a/examples/complete_cv_example.py
+++ b/examples/complete_cv_example.py
@@ -104,10 +104,11 @@ def training_function(config, args):
 
     # We need to initialize the trackers we use, and also store our configuration
     if args.with_tracking:
-        run = os.path.split(__file__)[-1].split(".")[0]
-        if args.logging_dir:
-            run = os.path.join(args.logging_dir, run)
-        accelerator.init_trackers(run, config)
+        if accelerator.is_main_process:
+            run = os.path.split(__file__)[-1].split(".")[0]
+            if args.logging_dir:
+                run = os.path.join(args.logging_dir, run)
+            accelerator.init_trackers(run, config)
 
     # Grab all the image filenames
     file_names = [os.path.join(args.data_dir, fname) for fname in os.listdir(args.data_dir) if fname.endswith(".jpg")]
diff --git a/examples/complete_nlp_example.py b/examples/complete_nlp_example.py
index a231566dc..abf85c8d9 100644
--- a/examples/complete_nlp_example.py
+++ b/examples/complete_nlp_example.py
@@ -81,10 +81,11 @@ def training_function(config, args):
 
     # We need to initialize the trackers we use, and also store our configuration
     if args.with_tracking:
-        run = os.path.split(__file__)[-1].split(".")[0]
-        if args.logging_dir:
-            run = os.path.join(args.logging_dir, run)
-        accelerator.init_trackers(run, config)
+        if accelerator.is_main_process:
+            run = os.path.split(__file__)[-1].split(".")[0]
+            if args.logging_dir:
+                run = os.path.join(args.logging_dir, run)
+            accelerator.init_trackers(run, config)
 
     tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
     datasets = load_dataset("glue", "mrpc")
@@ -245,9 +246,10 @@ def collate_fn(examples):
                 {
                     "accuracy": eval_metric["accuracy"],
                     "f1": eval_metric["f1"],
-                    "train_loss": total_loss,
+                    "train_loss": total_loss.item(),
                     "epoch": epoch,
-                }
+                },
+                step=epoch,
             )
 
         if checkpointing_steps == "epoch":
diff --git a/src/accelerate/tracking.py b/src/accelerate/tracking.py
index 218ddf2df..3efff77c9 100644
--- a/src/accelerate/tracking.py
+++ b/src/accelerate/tracking.py
@@ -136,8 +136,9 @@ def log(self, values: dict, step: Optional[int] = None):
         Logs `values` to the current run.
 
         Args:
-            values (Dictionary `str` to `str`, `float`, or `int`):
-                Values to be logged as key-value pairs. The values need to have type `str`, `float`, or `int`.
+            values (Dictionary `str` to `str`, `float`, `int` or `dict` of `str` to `float`/`int`):
+                Values to be logged as key-value pairs. The values need to have type `str`, `float`, `int` or `dict` of
+                `str` to `float`/`int`.
             step (`int`, *optional*):
                 The run step. If included, the log will be affiliated with this step.
         """
@@ -146,6 +147,8 @@ def log(self, values: dict, step: Optional[int] = None):
                 self.writer.add_scalar(k, v, global_step=step)
             elif isinstance(v, str):
                 self.writer.add_text(k, v, global_step=step)
+            elif isinstance(v, dict):
+                self.writer.add_scalars(k, v, global_step=step)
         self.writer.flush()
         logger.info("Successfully logged to TensorBoard")
 
@@ -170,7 +173,7 @@ class WandBTracker(GeneralTracker):
 
     def __init__(self, run_name: str):
         self.run_name = run_name
-        self.run = wandb.init(self.run_name)
+        self.run = wandb.init(project=self.run_name)
         logger.info(f"Initialized WandB project {self.run_name}")
         logger.info(
             "Make sure to log any initial configurations with `self.store_init_configuration` before training!"
@@ -193,8 +196,9 @@ def log(self, values: dict, step: Optional[int] = None):
         Logs `values` to the current run.
 
         Args:
-            values (Dictionary `str` to `str`, `float`, or `int`):
-                Values to be logged as key-value pairs. The values need to have type `str`, `float`, or `int`.
+            values (Dictionary `str` to `str`, `float`, `int` or `dict` of `str` to `float`/`int`):
+                Values to be logged as key-value pairs. The values need to have type `str`, `float`, `int` or `dict` of
+                `str` to `float`/`int`.
             step (`int`, *optional*):
                 The run step. If included, the log will be affiliated with this step.
         """
@@ -247,14 +251,21 @@ def log(self, values: dict, step: Optional[int] = None):
         Logs `values` to the current run.
 
         Args:
-            values (Dictionary `str` to `str`, `float`, or `int`):
-                Values to be logged as key-value pairs. The values need to have type `str`, `float`, or `int`.
+            values (Dictionary `str` to `str`, `float`, `int` or `dict` of `str` to `float`/`int`):
+                Values to be logged as key-value pairs. The values need to have type `str`, `float`, `int` or `dict` of
+                `str` to `float`/`int`.
             step (`int`, *optional*):
                 The run step. If included, the log will be affiliated with this step.
         """
         if step is not None:
             self.writer.set_step(step)
-        self.writer.log_others(values)
+        for k, v in values.items():
+            if isinstance(v, (int, float)):
+                self.writer.log_metric(k, v, step=step)
+            elif isinstance(v, str):
+                self.writer.log_other(k, v)
+            elif isinstance(v, dict):
+                self.writer.log_metrics(v, step=step)
         logger.info("Successfully logged to CometML")
 
     def finish(self):
diff --git a/tests/test_examples.py b/tests/test_examples.py
index 84b6adaef..b9759f8b5 100644
--- a/tests/test_examples.py
+++ b/tests/test_examples.py
@@ -149,9 +149,10 @@ def test_cv_examples(self):
             " " * 16 + "{\n\n",
             " " * 18 + '"accuracy": eval_metric["accuracy"],\n\n',
             " " * 18 + '"f1": eval_metric["f1"],\n\n',
-            " " * 18 + '"train_loss": total_loss,\n\n',
+            " " * 18 + '"train_loss": total_loss.item(),\n\n',
             " " * 18 + '"epoch": epoch,\n\n',
-            " " * 16 + "}\n",
+            " " * 16 + "},\n\n",
+            " " * 16 + "step=epoch,\n",
             " " * 8,
         ]
         self.one_complete_example("complete_cv_example.py", True, cv_path, special_strings)
diff --git a/tests/test_tracking.py b/tests/test_tracking.py
index e9a2170ee..f02db8d21 100644
--- a/tests/test_tracking.py
+++ b/tests/test_tracking.py
@@ -203,6 +203,9 @@ def get_value_from_key(log_list, key: str, is_param: bool = False):
             if "log_other" in j.keys():
                 if j["log_other"]["key"] == key:
                     return j["log_other"]["val"]
+            if "metric" in j.keys():
+                if j["metric"]["metricName"] == key:
+                    return j["metric"]["metricValue"]
 
     def test_init_trackers(self):
         with tempfile.TemporaryDirectory() as d:


Review comments: body: Do we really have either a Tensor or something else? Can't it always be `.item()` here?
diff_hunk: @@ -293,7 +294,7 @@ def collate_fn(examples):
                     {
                         "accuracy": eval_metric["accuracy"],
                         "f1": eval_metric["f1"],
-                        "train_loss": total_loss,
+                        "train_loss": total_loss.item() if type(total_loss) == torch.Tensor else total_loss,
body: Same comment
diff_hunk: @@ -208,15 +209,16 @@ def training_function(config, args):
 
         # New Code #
         # To actually log, we call `Accelerator.log`
-        # The values passed can be of `str`, `int`, or `float`
+        # The values passed can be of `str`, `int`, `float` or `dict` of `str` to `float`/`int`
         if args.with_tracking:
             accelerator.log(
                 {
                     "accuracy": eval_metric["accuracy"],
                     "f1": eval_metric["f1"],
-                    "train_loss": total_loss,
+                    "train_loss": total_loss.item() if type(total_loss) == torch.Tensor else total_loss,
body: Same here.
diff_hunk: @@ -245,9 +246,10 @@ def collate_fn(examples):
                 {
                     "accuracy": eval_metric["accuracy"],
                     "f1": eval_metric["f1"],
-                    "train_loss": total_loss,
+                    "train_loss": total_loss.item() if type(total_loss) == torch.Tensor else total_loss,

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: Oh, also make sure to edit the title of the PR ;-)