extremely slow training with multiple GPUs
I am training a  model with lightning where I am attempting to use all the GPUs on my system (4 in total).
My trainer is run as:
model = MyModel(hparams)
if torch.cuda.is_available():
    trainer = Trainer(gpus=-1)
else:
    trainer = Trainer()
trainer.fit(model)

My model is defined as follows:
class SiameseNet(pl.LightningModule):
    """
    Implement a siamese network as a feature extractor withh Lightning module
    """
    def __init__(self,
                 hparams):
        """
        Build the network
        """
        super(SiameseNet, self).__init__()
        self.net = self._build_net()
        self.hparams = hparams
        self.train_data_path = hparams.get('train_data_path', None)
        self.test_data_path = hparams.get('test_data_path', None)
        self.val_data_path = hparams.get('val_data_path', None)
        self.train_dataset = None
        self.val_dataset = None
        self.test_dataset = None

        self.lossfn = TripletLoss(margin=1.0)

    def forward_once(self, x):
        output = self.net(x)
        output = torch.squeeze(output)
        return output

    def forward(self, input1, input2, input3=None):
        output1 = self.forward_once(input1)
        output2 = self.forward_once(input2)

        if input3 is not None:
            output3 = self.forward_once(input3)
            return output1, output2, output3

        return output1, output2

    @staticmethod
    def _build_net():
        net = nn.Sequential(
            nn.Conv2d(3, 32,kernel_size=3,stride=2),
            nn.ReLU(inplace=True),
            nn.BatchNorm2d(32),

            nn.Conv2d(32, 64, kernel_size=3, stride=2),
            nn.ReLU(inplace=True),
            nn.BatchNorm2d(64),

            nn.Conv2d(64, 128, kernel_size=3, stride=2),
            nn.ReLU(inplace=True),
            nn.BatchNorm2d(128),

            nn.Conv2d(128, 256, kernel_size=1, stride=2),
            nn.ReLU(inplace=True),
            nn.BatchNorm2d(256),

            nn.Conv2d(256, 256, kernel_size=1, stride=2),
            nn.ReLU(inplace=True),
            nn.BatchNorm2d(256),

            nn.Conv2d(256, 512, kernel_size=3, stride=2),
            nn.ReLU(inplace=True),
            nn.BatchNorm2d(512),

            nn.Conv2d(512, 1024, kernel_size=1, stride=1),
            nn.ReLU(inplace=True),
            nn.BatchNorm2d(1024))

        return net

    def prepare_data(self):
        transform = torchvision.transforms.Compose([
            torchvision.transforms.Resize((128, 128)),
            torchvision.transforms.ColorJitter(hue=.05, saturation=.05),
            torchvision.transforms.RandomHorizontalFlip(),
            torchvision.transforms.RandomRotation(20, resample=PIL.Image.BILINEAR),
            torchvision.transforms.ToTensor()
        ])

        if self.train_data_path:
            train_folder_dataset = dset.ImageFolder(root=self.train_data_path)
            self.train_dataset = SiameseTriplet(image_folder_dataset=train_folder_dataset,
                                                transform=transform)
        if self.val_data_path:
            val_folder_dataset = dset.ImageFolder(root=self.val_data_path)
            self.val_dataset = SiameseTriplet(image_folder_dataset=val_folder_dataset)

        if self.test_data_path:
            test_folder_dataset = dset.ImageFolder(root=self.test_data_path)
            self.test_dataset = SiameseTriplet(image_folder_dataset=test_folder_dataset)

    def training_step(self, batch, batch_idx):
        anchor, positive, negative = batch
        anchor_out, positive_out, negative_out = self.forward(anchor, positive, negative)
        loss_val = self.lossfn(anchor_out, positive_out, negative_out)
        return {'loss': loss_val}

    def configure_optimizers(self):
        optimizer = optim.Adam(self.parameters(), lr=self.hparams.get('learning_rate', 0.001))
        scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)
        return [optimizer], [scheduler]

    @pl.data_loader
    def train_dataloader(self):
        if self.train_dataset:
            return DataLoader(self.train_dataset,
                              self.hparams.get('batch_size', 64),
                              num_workers=12)
        return None

When I try and run it, it seems the beginning of the epoch hangs for like 10 minutes to get data into the model and after that the progress is  very sluggish.
I also get these messages in  the beginning. Not sure if it is of concern
GPU available: True, used: True
No environment variable for node rank defined. Set as 0.
CUDA_VISIBLE_DEVICES: [0,1,2,3]
MASTER_ADDR environment variable is not defined. Set as localhost
initializing proc_rank 0 world 4
MASTER_ADDR environment variable is not defined. Set as localhost
initializing proc_rank 1 world 4
MASTER_ADDR environment variable is not defined. Set as localhost
initializing proc_rank 2 world 4
MASTER_ADDR environment variable is not defined. Set as localhost
initializing proc_rank 3 world 4

It basically hangs with this:
Epoch 1:   0%|                                                                                          | 0/172 [00:00<?, ?it/s]

During this time, looking at GPU  utilisation it seems:
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 418.87.00    Driver Version: 418.87.00    CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  GeForce GTX 108...  Off  | 00000000:05:00.0 Off |                  N/A |
| 48%   79C    P2    90W / 250W |   4527MiB / 11176MiB |    100%      Default |
+-------------------------------+----------------------+----------------------+
|   1  GeForce GTX 108...  Off  | 00000000:06:00.0 Off |                  N/A |
| 45%   76C    P2    85W / 250W |   1636MiB / 11178MiB |    100%      Default |
+-------------------------------+----------------------+----------------------+
|   2  GeForce GTX 108...  Off  | 00000000:09:00.0 Off |                  N/A |
| 45%   76C    P2    79W / 250W |   1626MiB / 11178MiB |    100%      Default |
+-------------------------------+----------------------+----------------------+
|   3  GeForce GTX 108...  Off  | 00000000:0A:00.0 Off |                  N/A |
| 32%   65C    P2    79W / 250W |   2689MiB / 11178MiB |    100%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|    0      1584      C   /home/pd/.conda/envs/alchera37/bin/python    801MiB |
|    0     10714      C   /home/pd/.conda/envs/alchera37/bin/python    543MiB |
|    0     28957      C   /home/pd/.conda/envs/alchera37/bin/python   1047MiB |
|    0     30880      C   /home/pd/.conda/envs/alchera37/bin/python   1063MiB |
|    0     32266      C   /home/pd/.conda/envs/alchera37/bin/python   1063MiB |
|    1     10733      C   /home/pd/.conda/envs/alchera37/bin/python    543MiB |
|    1     28972      C   /home/pd/.conda/envs/alchera37/bin/python   1063MiB |
|    2     10789      C   /home/pd/.conda/envs/alchera37/bin/python    543MiB |
|    2     32297      C   /home/pd/.conda/envs/alchera37/bin/python   1063MiB |
|    3     10807      C   /home/pd/.conda/envs/alchera37/bin/python    543MiB |
|    3     29006      C   /home/pd/.conda/envs/alchera37/bin/python   1063MiB |
|    3     30967      C   /home/pd/.conda/envs/alchera37/bin/python   1063MiB |
+-----------------------------------------------------------------------------+

So, it seems that getting the data into the GPU is quite slow even though everything looks maxed out.
And when it does eventually start the epoch after ~30 minutes, it seems to give similar performance as my CPU on MacBook Pro. I am really not sure if I am doing somethingvery  wrong here in how I am using  PL.
And then it gets stuck after this line (at least no console output)

This looks very familiar, and I am sure I fixed this problem in #2997, please try again with the latest version. Regarding the relative import error, you probably just launched the script in the wrong directory, but anyway I recommend to use absolute imports. Please let me know if the upgrade fixes your problem, thanks.