diff --git a/src/accelerate/tracking.py b/src/accelerate/tracking.py
index 839a7a2a9..028e431e1 100644
--- a/src/accelerate/tracking.py
+++ b/src/accelerate/tracking.py
@@ -16,9 +16,12 @@
 # Provide a project dir name, then each type of logger gets stored in project/{`logging_dir`}
 
 import os
+import time
 from abc import ABCMeta, abstractmethod, abstractproperty
 from typing import List, Optional, Union
 
+import yaml
+
 from .logging import get_logger
 from .utils import LoggerType, is_comet_ml_available, is_tensorboard_available, is_wandb_available
 
@@ -142,7 +145,8 @@ def tracker(self):
 
     def store_init_configuration(self, values: dict):
         """
-        Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment.
+        Logs `values` as hyperparameters for the run. Should be run at the beginning of your experiment. Stores the
+        hyperparameters in a yaml file for future use.
 
         Args:
             values (Dictionary `str` to `bool`, `str`, `float` or `int`):
@@ -151,7 +155,16 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        project_run_name = time.time()
+        dir_name = os.path.join(self.logging_dir, str(project_run_name))
+        os.makedirs(dir_name, exist_ok=True)
+        with open(os.path.join(dir_name, "hparams.yml"), "w") as outfile:
+            try:
+                yaml.dump(values, outfile)
+            except yaml.representer.RepresenterError:
+                logger.error("Serialization to store hyperparameters failed")
+                raise
+        logger.info("Stored initial configuration hyperparameters to TensorBoard and hparams yaml file")
 
     def log(self, values: dict, step: Optional[int] = None, **kwargs):
         """


Review comments: body: Perhaps it may be good to make use of the `logging_dir` to send the yaml to? Or should these be stored separately normally.

(This part):
```python
writer =  tensorboard.SummaryWriter(self.logging_dir, **kwargs)
```

diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("hparams.yaml", "w") as outfile:
body: So, should I save the yaml file in the `logging_dir`?
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("hparams.yaml", "w") as outfile:
body: Yep, as this is where users would expect it to wind up landing. E.g

logging_dir/hparams.yaml
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("hparams.yaml", "w") as outfile:
body: Yeah makes sense. I have made the changes now.
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("hparams.yaml", "w") as outfile:
body: Would we want to use pathlib `Path(self.logging_dir) / 'hparams.yaml'` or `os.path.join(self.logging_dir, 'hparams.yaml')` instead here for sanity purposes?
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("self.logging_dir/hparams.yaml", "w") as outfile:
body: Do we need to check if all keys/vals are serializable here, or are we going to let the error yaml throws be the one we present to users in this case? (not sure if this is done elsewhere in the codebase)
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("self.logging_dir/hparams.yaml", "w") as outfile:
+            yaml.dump(values, outfile)
body: We do that somewhat here: https://github.com/huggingface/accelerate/blob/main/src/accelerate/tracking.py#L170-L176

Maybe a try/catch that alerts the user that non-serializable bits were tried to be stored? Not sure what the try/catch error would be for that, otherwise I'm open to it just throwing the yaml error if we cannot :) 
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("self.logging_dir/hparams.yaml", "w") as outfile:
+            yaml.dump(values, outfile)
body: could catch a `yaml.representer.RepresenterError` and return the message maybe?
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("self.logging_dir/hparams.yaml", "w") as outfile:
+            yaml.dump(values, outfile)
body: Along with more descriptive explanation of what happened under the hood accelerate...otherwise its the same as letting yaml handle it. 

I'm fine with either solution, just wanted to bring it up!
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("self.logging_dir/hparams.yaml", "w") as outfile:
+            yaml.dump(values, outfile)
body: We should be careful here that `hparams.yaml` is saved separately for separate experiments. Should make sure that when you run 2 separate runs, you get 2 separate entries in the hparams dashboard of tensorboard. 

If we save directly to `logging_dir/hparams.yaml` we would be overwriting every time, eh? 
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("hparams.yaml", "w") as outfile:
body: I have changed the hardcoded path here.
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("self.logging_dir/hparams.yaml", "w") as outfile:
body: I have added the try/catch error. 
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("self.logging_dir/hparams.yaml", "w") as outfile:
+            yaml.dump(values, outfile)
body: I am not sure where to give a descriptive explanation. Should I add it as a comment or should I define it in the PR?
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("self.logging_dir/hparams.yaml", "w") as outfile:
+            yaml.dump(values, outfile)
body: The default tensorboard summary writer uses time.time() to create a folder for every run. So, I added the same for hparams.yml file. Hope this is okay.
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("self.logging_dir/hparams.yaml", "w") as outfile:
body: We should change this to actually raise the error, otherwise this will silently fail.

```suggestion
            except yaml.representer.RepresenterError:
                logger.error("Serialization to store hyperparameters failed")
                raise
```
I think just doing `logger.error` here is fine enough
diff_hunk: @@ -151,7 +154,13 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        project_run_name = time.time()
+        with open(os.path.join(self.logging_dir, project_run_name,  "hparams.yml"), "w") as outfile:
+            try:
+                yaml.dump(values, outfile)
+            except yaml.representer.RepresenterError:
+                logger.info("Serialization to store hyperparmeters failed")
body: Doing so in the `logger.error` (see my suggestions) is good enough :) 
diff_hunk: @@ -151,7 +153,9 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        with open("self.logging_dir/hparams.yaml", "w") as outfile:
+            yaml.dump(values, outfile)
body: Yes, I have made the changes.
diff_hunk: @@ -151,7 +154,13 @@ def store_init_configuration(self, values: dict):
         """
         self.writer.add_hparams(values, metric_dict={})
         self.writer.flush()
-        logger.info("Stored initial configuration hyperparameters to TensorBoard")
+        project_run_name = time.time()
+        with open(os.path.join(self.logging_dir, project_run_name,  "hparams.yml"), "w") as outfile:
+            try:
+                yaml.dump(values, outfile)
+            except yaml.representer.RepresenterError:
+                logger.info("Serialization to store hyperparmeters failed")

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: cc @nateraw if you have any comments 🤗 
body: @Shreyz-max cast `run_name` to a string on the yaml part and it should solve the error. E.g. `str(run_name)`

Great work on this, were almost there!
body: Yeah, sorry about the mistake. I did not notice this at all.
body: I suppose I need to create the folder.
body: Great job! CC @nateraw for one last look, but lg2m!