text: 
  
  TITLE
  Error with multi-gpu training 
  
  COMMENTS
  2
  
  REACTIONS
  +1: 0
  -1: 0
  laugh: 0
  hooray: 0
  heart: 0
  rocket: 0
  eyes: 0
  
  BODY
  I'm trying to build a QuestionAnswering model using transformers
  It works with single gpu training but fails with multiple gpus. 
  Is there any bug in the below code?
  
  
  ```
  class QAModel(pl.LightningModule):
      def __init__(self):
          super(QAModel, self).__init__()
          
          self.model_type = parameters["BaseModel_type"]
          self.config = AutoConfig.from_pretrained(model_name)
          self.base_model = AutoModelForQuestionAnswering.from_pretrained(model_name, config = self.config)
          self.tokenizer = tokenizer
          
          
      def forward(self,
                  input_ids=None,
                  attention_mask=None,
                  token_type_ids=None,
                  start_positions=None, 
                  end_positions=None):
  
          outputs = self.base_model(
              input_ids,
              attention_mask=attention_mask,
              token_type_ids=token_type_ids,
              start_positions=start_positions, 
              end_positions=end_positions,
          )
  
          return outputs
  
      
      def prepare_data(self):
          self.train_dataset, _, _ = load_data(parameters["TRAIN_FILE"], is_training=True)
                                                                                  
          self.val_dataset, self.val_examples, self.val_features = load_data(parameters["DEV_FILE"], is_training=False)
                                                                                  
          self.test_dataset, self.test_examples, self.test_features = load_data(parameters["TEST_FILE"], is_training=False)
  
  
      def train_dataloader(self):
          return DataLoader(dataset=self.train_dataset, batch_size=parameters["batch_size"], shuffle=True, num_workers=parameters["num_threads"])
  
      def val_dataloader(self):
          return DataLoader(dataset=self.val_dataset, batch_size=parameters["batch_size"], num_workers=parameters["num_threads"])
  
      def test_dataloader(self):
          return DataLoader(dataset=self.test_dataset, batch_size=parameters["batch_size"], num_workers=parameters["num_threads"])
  
      def configure_optimizers(self):
          return torch.optim.Adam(self.parameters(), lr=parameters["learning_rate"])
      
      def training_step(self, batch, batch_idx):
          inputs = {
                  "input_ids": batch[0],
                  "attention_mask": batch[1],
                  "token_type_ids": batch[2],
                  "start_positions": batch[3],
                  "end_positions": batch[4],
              }
          outputs = self.forward(**inputs)        
          loss = outputs[0]
          return {"loss": loss}    
  
  
      def validation_step(self, batch, batch_idx):
          inputs = {
                  "input_ids": batch[0],
                  "attention_mask": batch[1],
                  "token_type_ids": batch[2],
              }
  
      
          feature_indices = batch[3]
  
          outputs = self.forward(**inputs)        
                         
  
  model = QAModel()
  trainer = pl.Trainer(gpus=-1, distributed_backend='dp', max_epochs=parameters["epochs"])
  
  trainer.fit(model)
  ```
  
  I get this error on running it with multiple gpus:
  ```
  RuntimeError: grad can be implicitly created only for scalar outputs
  ```