Repository: Lightning-AI/pytorch-lightning

Issue:
In Multi GPU DDP, pytorch-lightning creates several tfevents files
**Describe the bug**

Right now pytorch-lightning seems to create several tfevent files in the multi-gpu ddp way:
e.g. for 2 GPUs:
```
-rw-rw-r--. 1 sam sam   40 Sep 19 08:11 events.out.tfevents.1568880714.google2-compute82.3156.0
-rw-rw-r--. 1 sam sam 165K Sep 19 08:22 events.out.tfevents.1568880716.google2-compute82.3186.0
-rw-rw-r--. 1 sam sam   40 Sep 19 08:11 events.out.tfevents.1568880718.google2-compute82.3199.0
```
I suppose the first one is created by the main process and the next 2 are created by the 2 DDP processes (one per GPU). Unfortunately, the actual events are not logged in the last created one, and that confuses tensorboard, cf https://github.com/tensorflow/tensorboard/issues/1011 

I have to restart tensorboard if I want to see the new data.


A clear and concise description of what the bug is.

**To Reproduce**
Launch any training on multi GPU DDP.

**Expected behavior**
Only one tfevent file is created, from the master GPU.



Discussion:
thanks for bringing this up. this has been reported a few times already. the problem is what you described. 

the solution is to init the logger from proc zero only. want to take a look at how we can approach this? @neggert is working on an abstraction that will need this fix
Are you thinking we should make sure that the test tube experiment doesn't even get initialized unless we're on process 0? Right now I have it initialize, but never log, but that's easy enough to change.
yeah, i think the best thing is to make sure it’s only initialized once. This will save a ton of space in the experiment file as well

Resolving patch:
diff --git a/pytorch_lightning/logging/mlflow_logger.py b/pytorch_lightning/logging/mlflow_logger.py
--- a/pytorch_lightning/logging/mlflow_logger.py
+++ b/pytorch_lightning/logging/mlflow_logger.py
@@ -9,21 +9,28 @@
 
 
 class MLFlowLogger(LightningLoggerBase):
-
     def __init__(self, experiment_name, tracking_uri=None):
         super().__init__()
         self.client = mlflow.tracking.MlflowClient(tracking_uri)
+        self.experiment_name = experiment_name
+        self._run_id = None
+
+    @property
+    def run_id(self):
+        if self._run_id is not None:
+            return self._run_id
 
-        experiment = self.client.get_experiment_by_name(experiment_name)
+        experiment = self.client.get_experiment_by_name(self.experiment_name)
         if experiment is None:
             logger.warning(
-                f"Experiment with name f{experiment_name} not found. Creating it."
+                f"Experiment with name f{self.experiment_name} not found. Creating it."
             )
-            self.client.create_experiment(experiment_name)
-            experiment = self.client.get_experiment_by_name(experiment_name)
+            self.client.create_experiment(self.experiment_name)
+            experiment = self.client.get_experiment_by_name(self.experiment_name)
 
         run = self.client.create_run(experiment.experiment_id)
-        self.run_id = run.info.run_id
+        self._run_id = run.info.run_id
+        return self._run_id
 
     @rank_zero_only
     def log_hyperparams(self, params):
diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py
--- a/pytorch_lightning/trainer/trainer.py
+++ b/pytorch_lightning/trainer/trainer.py
@@ -890,8 +890,9 @@ def __run_pretrain_routine(self, model):
             ref_model.logger = self.logger
 
             # save exp to get started
-            if self.proc_rank == 0:
-                self.logger.save()
+            if hasattr(ref_model, "hparams"):
+                self.logger.log_hyperparams(ref_model.hparams)
+            self.logger.save()
 
         # track model now.
         # if cluster resets state, the model will update with the saved weights


Test patch:
diff --git a/pytorch_lightning/logging/test_tube_logger.py b/pytorch_lightning/logging/test_tube_logger.py
--- a/pytorch_lightning/logging/test_tube_logger.py
+++ b/pytorch_lightning/logging/test_tube_logger.py
@@ -11,13 +11,26 @@ def __init__(
         self, save_dir, name="default", debug=False, version=None, create_git_tag=False
     ):
         super().__init__()
-        self.experiment = Experiment(
-            save_dir=save_dir,
-            name=name,
-            debug=debug,
-            version=version,
-            create_git_tag=create_git_tag,
+        self.save_dir = save_dir
+        self.name = name
+        self.debug = debug
+        self._version = version
+        self.create_git_tag = create_git_tag
+        self._experiment = None
+
+    @property
+    def experiment(self):
+        if self._experiment is not None:
+            return self._experiment
+        self._experiment = Experiment(
+            save_dir=self.save_dir,
+            name=self.name,
+            debug=self.debug,
+            version=self.version,
+            create_git_tag=self.create_git_tag,
+            rank=self.rank,
         )
+        return self._experiment
 
     @rank_zero_only
     def log_hyperparams(self, params):
@@ -41,15 +54,24 @@ def close(self):
 
     @property
     def rank(self):
-        return self.experiment.rank
+        if self._experiment is None:
+            return self._rank
+        else:
+            return self.experiment.rank
 
     @rank.setter
     def rank(self, value):
-        self.experiment.rank = value
+        if self._experiment is None:
+            self._rank = value
+        else:
+            return self.experiment.rank
 
     @property
     def version(self):
-        return self.experiment.version
+        if self._experiment is None:
+            return self._version
+        else:
+            return self.experiment.version
 
     # Test tube experiments are not pickleable, so we need to override a few
     # methods to get DDP working. See
@@ -57,10 +79,10 @@ def version(self):
     # for more info.
     def __getstate__(self):
         state = self.__dict__.copy()
-        state["experiment"] = self.experiment.get_meta_copy()
+        state["_experiment"] = self.experiment.get_meta_copy()
         return state
 
     def __setstate__(self, state):
-        self.experiment = state["experiment"].get_non_ddp_exp()
-        del state['experiment']
+        self._experiment = state["_experiment"].get_non_ddp_exp()
+        del state["_experiment"]
         self.__dict__.update(state)
diff --git a/tests/test_logging.py b/tests/test_logging.py
--- a/tests/test_logging.py
+++ b/tests/test_logging.py
@@ -19,8 +19,6 @@ def test_testtube_logger():
     save_dir = init_save_dir()
 
     logger = get_test_tube_logger(False)
-    logger.log_hyperparams(hparams)
-    logger.save()
 
     trainer_options = dict(
         max_nb_epochs=1,