# Buggy code
def training_forward(self, batch, batch_idx, opt_idx, hiddens):
    """
    Handle forward for each training case (distributed, single gpu, etc...)
    :param batch:
    :param batch_idx:
    :return:
    """
    # ---------------
    # FORWARD
    # ---------------
    # enable not needing to add opt_idx to training_step
    args = [batch, batch_idx]

    if len(self.optimizers) > 1:
        if self.has_arg("training_step", "optimizer_idx"):
            args.append(opt_idx)
        else:
            raise ValueError(
                f"Your LightningModule defines {len(self.optimizers)} optimizers but "
                f'training_step is missing the "optimizer_idx" argument.'
            )

    # pass hiddens if using tbptt
    if self.truncated_bptt_steps is not None:
        args.append(hiddens)

    # distributed forward
    if self.use_ddp or self.use_ddp2 or self.use_dp:
        output = self.model(*args)

    # single GPU forward
    elif self.single_gpu:
        gpu_id = 0
        if isinstance(self.data_parallel_device_ids, list):
            gpu_id = self.data_parallel_device_ids[0]
        batch = self.transfer_batch_to_gpu(batch.copy(), gpu_id)
        args[0] = batch
        output = self.model.training_step(*args)

    # CPU forward
    else:
        output = self.model.training_step(*args)

    # allow any mode to define training_end
    if self.is_overriden("training_end"):
        model_ref = self.get_model()
        output = model_ref.training_end(output)

    # format and reduce outputs accordingly
    output = self.process_output(output, train=True)

    return output


# Traceback (AttributeError)
Traceback (most recent call last):
File "scripts/msmacro.py", line 113, in <module>
main()
File "scripts/msmacro.py", line 109, in main
trainer.fit(model)
File "/home/zhaohao/Documents/pytorch-lightning/pytorch_lightning/trainer/trainer.py", line 405, in fit
self.single_gpu_train(model)
File "/home/zhaohao/Documents/pytorch-lightning/pytorch_lightning/trainer/distrib_parts.py", line 441, in single_gpu_train
self.run_pretrain_routine(model)
File "/home/zhaohao/Documents/pytorch-lightning/pytorch_lightning/trainer/trainer.py", line 539, in run_pretrain_routine
self.train()
File "/home/zhaohao/Documents/pytorch-lightning/pytorch_lightning/trainer/training_loop.py", line 332, in train
self.run_training_epoch()
File "/home/zhaohao/Documents/pytorch-lightning/pytorch_lightning/trainer/training_loop.py", line 385, in run_training_epoch
output = self.run_training_batch(batch, batch_idx)
File "/home/zhaohao/Documents/pytorch-lightning/pytorch_lightning/trainer/training_loop.py", line 504, in run_training_batch
loss = optimizer_closure()
File "/home/zhaohao/Documents/pytorch-lightning/pytorch_lightning/trainer/training_loop.py", line 473, in optimizer_closure
split_batch, batch_idx, opt_idx, self.hiddens)
File "/home/zhaohao/Documents/pytorch-lightning/pytorch_lightning/trainer/training_loop.py", line 589, in training_forward
batch = self.transfer_batch_to_gpu(batch.copy(), gpu_id)
AttributeError: 'tuple' object has no attribute 'copy'

# Fixed code
def training_forward(self, batch, batch_idx, opt_idx, hiddens):
    """
    Handle forward for each training case (distributed, single gpu, etc...)
    :param batch:
    :param batch_idx:
    :return:
    """
    # ---------------
    # FORWARD
    # ---------------
    # enable not needing to add opt_idx to training_step
    args = [batch, batch_idx]

    if len(self.optimizers) > 1:
        if self.has_arg("training_step", "optimizer_idx"):
            args.append(opt_idx)
        else:
            raise ValueError(
                f"Your LightningModule defines {len(self.optimizers)} optimizers but "
                f'training_step is missing the "optimizer_idx" argument.'
            )

    # pass hiddens if using tbptt
    if self.truncated_bptt_steps is not None:
        args.append(hiddens)

    # distributed forward
    if self.use_ddp or self.use_ddp2 or self.use_dp:
        output = self.model(*args)

    # single GPU forward
    elif self.single_gpu:
        gpu_id = 0
        if isinstance(self.data_parallel_device_ids, list):
            gpu_id = self.data_parallel_device_ids[0]
        batch = self.transfer_batch_to_gpu(copy.copy(batch), gpu_id)
        args[0] = batch
        output = self.model.training_step(*args)

    # CPU forward
    else:
        output = self.model.training_step(*args)

    # allow any mode to define training_end
    if self.is_overriden("training_end"):
        model_ref = self.get_model()
        output = model_ref.training_end(output)

    # format and reduce outputs accordingly
    output = self.process_output(output, train=True)

    return output