Repository: Lightning-AI/pytorch-lightning

Issue:
'NoneType' object has no attribute 'lower'  while training on TPU
## 🐛 Bug

distributed_backend is not set to 'tpu' which breaks it here: line https://github.com/PyTorchLightning/pytorch-lightning/blob/2c935d048e69a2890889dea768ecceb0252cf321/pytorch_lightning/trainer/distrib_data_parallel.py#L412


distributed_backend has to be explicitly specified in Trainer params to have it working which is misleading from the docs: https://pytorch-lightning.readthedocs.io/en/latest/tpu.html#distributed-backend-with-tpu


### To Reproduce

Steps to reproduce the behavior:

https://www.kaggle.com/lezwon/lightning-mnist-tpu


### Expected behavior

Should automatically set `distributed_backend` and train successfully.

### Additional context

Related to : 
https://github.com/PyTorchLightning/pytorch-lightning/issues/2698#issuecomment-671665202
https://github.com/PyTorchLightning/pytorch-lightning/issues/2812#issuecomment-668659590



Discussion:

Resolving patch:
diff --git a/pytorch_lightning/trainer/distrib_data_parallel.py b/pytorch_lightning/trainer/distrib_data_parallel.py
--- a/pytorch_lightning/trainer/distrib_data_parallel.py
+++ b/pytorch_lightning/trainer/distrib_data_parallel.py
@@ -238,9 +238,6 @@ def is_function_implemented(self, *args) -> bool:
         """Warning: this is just empty shell for code implemented in other class."""
 
     def init_tpu(self):
-        # turn off all the GPU stuff
-        # self.distributed_backend = 'tpu'
-
         # enable tpu
         self.use_tpu = True
 
diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py
--- a/pytorch_lightning/trainer/trainer.py
+++ b/pytorch_lightning/trainer/trainer.py
@@ -520,6 +520,7 @@ def __init__(
 
         # override dist backend when using tpus
         if self.on_tpu:
+            self.distributed_backend = 'tpu'
             self.init_tpu()
 
         # init flags for SLURM+DDP to work


Test patch:
diff --git a/tests/models/test_tpu.py b/tests/models/test_tpu.py
--- a/tests/models/test_tpu.py
+++ b/tests/models/test_tpu.py
@@ -41,7 +41,6 @@ def test_model_tpu_cores_1(tmpdir):
         default_root_dir=tmpdir,
         progress_bar_refresh_rate=0,
         max_epochs=1,
-        distributed_backend='tpu',
         tpu_cores=1,
         limit_train_batches=0.4,
         limit_val_batches=0.4,
@@ -60,7 +59,6 @@ def test_model_tpu_index(tmpdir, tpu_core):
         default_root_dir=tmpdir,
         progress_bar_refresh_rate=0,
         max_epochs=1,
-        distributed_backend='tpu',
         tpu_cores=[tpu_core],
         limit_train_batches=0.4,
         limit_val_batches=0.4,
@@ -79,7 +77,6 @@ def test_model_tpu_cores_8(tmpdir):
         default_root_dir=tmpdir,
         progress_bar_refresh_rate=0,
         max_epochs=1,
-        distributed_backend='tpu',
         tpu_cores=8,
         limit_train_batches=0.4,
         limit_val_batches=0.4,
@@ -102,7 +99,6 @@ def test_model_16bit_tpu_cores_1(tmpdir):
         precision=16,
         progress_bar_refresh_rate=0,
         max_epochs=1,
-        distributed_backend='tpu',
         tpu_cores=1,
         limit_train_batches=0.4,
         limit_val_batches=0.4,
@@ -125,7 +121,6 @@ def test_model_16bit_tpu_index(tmpdir, tpu_core):
         train_percent_check=0.4,
         val_percent_check=0.2,
         max_epochs=1,
-        distributed_backend='tpu',
         tpu_cores=[tpu_core],
         limit_train_batches=0.4,
         limit_val_batches=0.4,
@@ -146,7 +141,6 @@ def test_model_16bit_tpu_cores_8(tmpdir):
         precision=16,
         progress_bar_refresh_rate=0,
         max_epochs=1,
-        distributed_backend='tpu',
         tpu_cores=8,
         limit_train_batches=0.4,
         limit_val_batches=0.4,
@@ -172,7 +166,6 @@ def test_model_tpu_early_stop(tmpdir):
         max_epochs=50,
         limit_train_batches=10,
         limit_val_batches=10,
-        distributed_backend='tpu',
         tpu_cores=1,
     )
     trainer.fit(model)
@@ -186,7 +179,6 @@ def test_tpu_grad_norm(tmpdir):
         default_root_dir=tmpdir,
         progress_bar_refresh_rate=0,
         max_epochs=1,
-        distributed_backend='tpu',
         tpu_cores=1,
         limit_train_batches=0.4,
         limit_val_batches=0.4,
@@ -204,7 +196,11 @@ def test_dataloaders_passed_to_fit(tmpdir):
 
     model = EvalModelTemplate()
 
-    trainer = Trainer(default_root_dir=tmpdir, max_epochs=1, distributed_backend='tpu', tpu_cores=8)
+    trainer = Trainer(
+        default_root_dir=tmpdir,
+        max_epochs=1,
+        tpu_cores=8
+    )
     result = trainer.fit(model, train_dataloader=model.train_dataloader(), val_dataloaders=model.val_dataloader())
     assert result, "TPU doesn't work with dataloaders passed to fit()."
 
@@ -221,9 +217,7 @@ def test_tpu_id_to_be_as_expected(tpu_cores, expected_tpu_id):
 def test_tpu_misconfiguration():
     """Test if trainer.tpu_id is set as expected"""
     with pytest.raises(MisconfigurationException, match="`tpu_cores` can only be"):
-        Trainer(
-            tpu_cores=[1, 8], distributed_backend='tpu',
-        )
+        Trainer(tpu_cores=[1, 8])
 
 
 # @patch('pytorch_lightning.trainer.trainer.XLA_AVAILABLE', False)
@@ -236,9 +230,14 @@ def test_exception_when_no_tpu_found(tmpdir):
         max_epochs=1,
         train_percent_check=0.4,
         val_percent_check=0.2,
-        distributed_backend='tpu',
         tpu_cores=8,
     )
 
     with pytest.raises(MisconfigurationException, match='PyTorch XLA not installed.'):
         trainer.fit(model)
+
+
+@pytest.mark.parametrize('tpu_cores', [1, 8, [1]])
+def test_distributed_backend_set_when_using_tpu(tmpdir, tpu_cores):
+    """Test if distributed_backend is set to `tpu` when tpu_cores is not None"""
+    assert Trainer(tpu_cores=tpu_cores).distributed_backend == 'tpu'