Repository: Lightning-AI/pytorch-lightning

Issue:
Remove SLURM warning when in interactive mode
No need to show the "we launched processes for you" warning when in interactive SLURM mode.

Suggested by @adefazio 


Discussion:

Resolving patch:
diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py
--- a/pytorch_lightning/trainer/trainer.py
+++ b/pytorch_lightning/trainer/trainer.py
@@ -706,14 +706,6 @@ def fit(self, model):
             else:
                 nb_gpus = self.nb_requested_gpus
                 nb_tasks = self.nb_slurm_tasks
-                msg = f"""
-                You requested {nb_gpus}s GPUs but launched {nb_tasks}s slurm tasks.
-                We will launch {nb_gpus}s processes for you.
-                We recommend you let slurm manage the processes by setting:
-                --ntasks-per-node={nb_gpus}s
-                If you're not using SLURM, ignore this message!
-                """
-                warnings.warn(msg)
                 mp.spawn(self.ddp_train, nprocs=self.num_gpus, args=(model, ))
 
         # 1 gpu or dp option triggers training using DP module
@@ -1272,7 +1264,8 @@ def __run_training_batch(self, batch, batch_nb):
             # wrap the forward step in a closure so second order methods work
             def optimizer_closure():
                 # forward pass
-                closure_loss, model_specific_tqdm_metrics = self.__training_forward(batch, batch_nb, opt_idx)
+                output = self.__training_forward(batch, batch_nb, opt_idx)
+                closure_loss, model_specific_tqdm_metrics = output
 
                 # track metrics
                 self.__add_tqdm_metrics(model_specific_tqdm_metrics)
@@ -1320,7 +1313,8 @@ def optimizer_closure():
                 # calls .step(), .zero_grad()
                 # override function to modify this behavior
                 model = self.__get_model()
-                model.optimizer_step(self.current_epoch, batch_nb, optimizer, opt_idx, optimizer_closure)
+                model.optimizer_step(self.current_epoch, batch_nb,
+                                     optimizer, opt_idx, optimizer_closure)
 
                 # calculate running loss for display
                 self.running_loss.append(self.batch_loss_value)


Test patch:
diff --git a/pytorch_lightning/testing/lm_test_module_base.py b/pytorch_lightning/testing/lm_test_module_base.py
--- a/pytorch_lightning/testing/lm_test_module_base.py
+++ b/pytorch_lightning/testing/lm_test_module_base.py
@@ -115,11 +115,11 @@ def training_step(self, batch, batch_idx):
     # ---------------------
     def configure_optimizers(self):
         """
-        return whatever optimizers we want here
+        return whatever optimizers we want here.
         :return: list of optimizers
         """
         # try no scheduler for this model (testing purposes)
-        if self.hparams.optimizer == 'lbfgs':
+        if self.hparams.optimizer_name == 'lbfgs':
             optimizer = optim.LBFGS(self.parameters(), lr=self.hparams.learning_rate)
         else:
             optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
diff --git a/tests/test_logging.py b/tests/test_logging.py
--- a/tests/test_logging.py
+++ b/tests/test_logging.py
@@ -3,15 +3,20 @@
 import shutil
 
 import numpy as np
+import torch
 from pytorch_lightning import Trainer
 
 from pytorch_lightning.testing import LightningTestModel
-
 from .test_models import get_hparams, get_test_tube_logger, init_save_dir, clear_save_dir
 
+RANDOM_SEEDS = list(np.random.randint(0, 10000, 1000))
+
 
 def test_testtube_logger():
-    """verify that basic functionality of test tube logger works"""
+    """
+    verify that basic functionality of test tube logger works
+    """
+    reset_seed()
 
     hparams = get_hparams()
     model = LightningTestModel(hparams)
@@ -35,7 +40,11 @@ def test_testtube_logger():
 
 
 def test_testtube_pickle():
-    """Verify that pickling a trainer containing a test tube logger works"""
+    """
+    Verify that pickling a trainer containing a test tube logger works
+    """
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams)
 
@@ -58,7 +67,11 @@ def test_testtube_pickle():
 
 
 def test_mlflow_logger():
-    """verify that basic functionality of mlflow logger works"""
+    """
+    verify that basic functionality of mlflow logger works
+    """
+    reset_seed()
+
     try:
         from pytorch_lightning.logging import MLFlowLogger
     except ModuleNotFoundError:
@@ -90,7 +103,11 @@ def test_mlflow_logger():
 
 
 def test_mlflow_pickle():
-    """verify that pickling trainer with mlflow logger works"""
+    """
+    verify that pickling trainer with mlflow logger works
+    """
+    reset_seed()
+
     try:
         from pytorch_lightning.logging import MLFlowLogger
     except ModuleNotFoundError:
@@ -115,3 +132,9 @@ def test_mlflow_pickle():
     pkl_bytes = pickle.dumps(trainer)
     trainer2 = pickle.loads(pkl_bytes)
     trainer2.logger.log_metrics({"acc": 1.0})
+
+
+def reset_seed():
+    SEED = RANDOM_SEEDS.pop()
+    torch.manual_seed(SEED)
+    np.random.seed(SEED)
diff --git a/tests/test_models.py b/tests/test_models.py
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -31,9 +31,7 @@
 from pytorch_lightning.logging import TestTubeLogger
 from examples import LightningTemplateModel
 
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
+RANDOM_SEEDS = list(np.random.randint(0, 10000, 1000))
 
 
 # ------------------------------------------------------------------------
@@ -44,6 +42,8 @@ def test_default_logger_callbacks_cpu_model():
     Test each of the trainer options
     :return:
     """
+    reset_seed()
+
     trainer_options = dict(
         max_nb_epochs=1,
         gradient_clip_val=1.0,
@@ -67,14 +67,16 @@ def test_lbfgs_cpu_model():
     Test each of the trainer options
     :return:
     """
+    reset_seed()
+
     trainer_options = dict(
         max_nb_epochs=1,
         gradient_clip_val=1.0,
         overfit_pct=0.20,
         print_nan_grads=True,
         show_progress_bar=False,
-        train_percent_check=0.01,
-        val_percent_check=0.01
+        train_percent_check=0.1,
+        val_percent_check=0.1
     )
 
     model, hparams = get_model(use_test_model=True, lbfgs=True)
@@ -84,6 +86,7 @@ def test_lbfgs_cpu_model():
     model.freeze()
     model.unfreeze()
 
+
 def test_multi_gpu_model_ddp2():
     """
     Make sure DDP2 works
@@ -93,6 +96,9 @@ def test_multi_gpu_model_ddp2():
         return
 
     os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0])
+
+    reset_seed()
+
     model, hparams = get_model()
     trainer_options = dict(
         show_progress_bar=True,
@@ -115,6 +121,8 @@ def test_dp_resume():
     if not can_run_gpu_test():
         return
 
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams)
 
@@ -195,6 +203,8 @@ def test_running_test_pretrained_model_ddp():
     if not can_run_gpu_test():
         return
 
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams)
 
@@ -240,6 +250,8 @@ def test_running_test_pretrained_model_ddp():
 
 def test_running_test_after_fitting():
     """Verify test() on fitted model"""
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams)
 
@@ -278,6 +290,8 @@ def test_running_test_after_fitting():
 
 
 def test_running_test_without_val():
+    reset_seed()
+
     """Verify test() works on a model with no val_loader"""
     class CurrentTestModel(LightningTestMixin, LightningTestModelBase):
         pass
@@ -319,6 +333,8 @@ class CurrentTestModel(LightningTestMixin, LightningTestModelBase):
 
 
 def test_running_test_pretrained_model():
+    reset_seed()
+
     """Verify test() on pretrained model"""
     hparams = get_hparams()
     model = LightningTestModel(hparams)
@@ -361,6 +377,8 @@ def test_running_test_pretrained_model():
 
 
 def test_running_test_pretrained_model_dp():
+    reset_seed()
+
     """Verify test() on pretrained model"""
     if not can_run_gpu_test():
         return
@@ -407,6 +425,8 @@ def test_running_test_pretrained_model_dp():
 
 
 def test_gradient_accumulation_scheduling():
+    reset_seed()
+
     """
     Test grad accumulation by the freq of optimizer updates
     """
@@ -487,6 +507,8 @@ def test_multi_gpu_model_ddp():
         return
 
     os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0])
+    reset_seed()
+
     model, hparams = get_model()
     trainer_options = dict(
         show_progress_bar=False,
@@ -501,6 +523,7 @@ def test_multi_gpu_model_ddp():
 
 
 def test_optimizer_return_options():
+    reset_seed()
 
     trainer = Trainer()
     model, hparams = get_model()
@@ -531,6 +554,8 @@ def test_optimizer_return_options():
 
 
 def test_single_gpu_batch_parse():
+    reset_seed()
+
     if not can_run_gpu_test():
         return
 
@@ -577,6 +602,7 @@ def test_early_stopping_cpu_model():
     Test each of the trainer options
     :return:
     """
+    reset_seed()
 
     stopping = EarlyStopping(monitor='val_loss')
     trainer_options = dict(
@@ -604,6 +630,8 @@ def test_no_val_module():
     Tests use case where trainer saves the model, and user loads it from tags independently
     :return:
     """
+    reset_seed()
+
     hparams = get_hparams()
 
     class CurrentTestModel(LightningTestModelBase):
@@ -650,6 +678,7 @@ def test_no_val_end_module():
     Tests use case where trainer saves the model, and user loads it from tags independently
     :return:
     """
+    reset_seed()
 
     class CurrentTestModel(LightningValidationStepMixin, LightningTestModelBase):
         pass
@@ -696,6 +725,8 @@ def test_simple_cpu():
     Verify continue training session on CPU
     :return:
     """
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams)
 
@@ -723,6 +754,8 @@ def test_amp_single_gpu():
     Make sure DDP + AMP work
     :return:
     """
+    reset_seed()
+
     if not torch.cuda.is_available():
         warnings.warn('test_amp_gpu_ddp cannot run.'
                       'Rerun on a GPU node to run this test')
@@ -751,6 +784,8 @@ def test_no_amp_single_gpu():
     Make sure DDP + AMP work
     :return:
     """
+    reset_seed()
+
     if not torch.cuda.is_available():
         warnings.warn('test_amp_gpu_ddp cannot run.'
                       'Rerun on a GPU node to run this test')
@@ -780,6 +815,8 @@ def test_cpu_restore_training():
     Verify continue training session on CPU
     :return:
     """
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams)
 
@@ -852,6 +889,8 @@ def test_amp_gpu_ddp():
 
     os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0])
 
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams)
 
@@ -871,6 +910,8 @@ def test_cpu_slurm_save_load():
     Verify model save/load/checkpoint on CPU
     :return:
     """
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams)
 
@@ -946,6 +987,8 @@ def assert_pred_same():
 
 
 def test_loading_meta_tags():
+    reset_seed()
+
     from argparse import Namespace
     hparams = get_hparams()
 
@@ -967,6 +1010,7 @@ def test_loading_meta_tags():
 
 
 def test_dp_output_reduce():
+    reset_seed()
 
     # test identity when we have a single gpu
     out = torch.rand(3, 1)
@@ -992,6 +1036,8 @@ def test_model_saving_loading():
     Tests use case where trainer saves the model, and user loads it from tags independently
     :return:
     """
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams)
 
@@ -1047,6 +1093,8 @@ def test_model_saving_loading():
 
 
 def test_model_freeze_unfreeze():
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams)
 
@@ -1066,6 +1114,8 @@ def test_amp_gpu_ddp_slurm_managed():
     os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0])
     os.environ['SLURM_LOCALID'] = str(0)
 
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams)
 
@@ -1132,6 +1182,7 @@ def test_cpu_model_with_amp():
     Make sure model trains on CPU
     :return:
     """
+    reset_seed()
 
     trainer_options = dict(
         show_progress_bar=False,
@@ -1153,6 +1204,7 @@ def test_cpu_model():
     Make sure model trains on CPU
     :return:
     """
+    reset_seed()
 
     trainer_options = dict(
         show_progress_bar=False,
@@ -1172,6 +1224,7 @@ def test_all_features_cpu_model():
     Test each of the trainer options
     :return:
     """
+    reset_seed()
 
     trainer_options = dict(
         gradient_clip_val=1.0,
@@ -1195,6 +1248,8 @@ def test_single_gpu_model():
     Make sure single GPU works (DP mode)
     :return:
     """
+    reset_seed()
+
     if not torch.cuda.is_available():
         warnings.warn('test_single_gpu_model cannot run.'
                       ' Rerun on a GPU node to run this test')
@@ -1218,6 +1273,8 @@ def test_multi_gpu_none_backend():
     distributed_backend = None
     :return:
     """
+    reset_seed()
+
     if not can_run_gpu_test():
         return
 
@@ -1239,6 +1296,8 @@ def test_multi_gpu_model_dp():
     Make sure DP works
     :return:
     """
+    reset_seed()
+
     if not can_run_gpu_test():
         return
 
@@ -1263,6 +1322,8 @@ def test_amp_gpu_dp():
     Make sure DP + AMP work
     :return:
     """
+    reset_seed()
+
     if not can_run_gpu_test():
         return
 
@@ -1287,6 +1348,8 @@ def test_ddp_sampler_error():
 
     os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0])
 
+    reset_seed()
+
     hparams = get_hparams()
     model = LightningTestModel(hparams, force_remove_distributed_sampler=True)
 
@@ -1313,6 +1376,8 @@ def test_multiple_val_dataloader():
     Verify multiple val_dataloader
     :return:
     """
+    reset_seed()
+
     class CurrentTestModel(
         LightningValidationMultipleDataloadersMixin,
         LightningTestModelBase
@@ -1348,6 +1413,8 @@ def test_multiple_test_dataloader():
     Verify multiple test_dataloader
     :return:
     """
+    reset_seed()
+
     class CurrentTestModel(
         LightningTestMultipleDataloadersMixin,
         LightningTestModelBase
@@ -1472,7 +1539,7 @@ def get_model(use_test_model=False, lbfgs=False):
     # set up model with these hyperparams
     hparams = get_hparams()
     if lbfgs:
-        hparams.optimizer = 'lbfgs'
+        setattr(hparams, 'optimizer_name', 'lbfgs')
 
     if use_test_model:
         model = LightningTestModel(hparams)
@@ -1577,5 +1644,11 @@ def can_run_gpu_test():
     return True
 
 
+def reset_seed():
+    SEED = RANDOM_SEEDS.pop()
+    torch.manual_seed(SEED)
+    np.random.seed(SEED)
+
+
 if __name__ == '__main__':
     pytest.main([__file__])