# Buggy code
def __call__(self, value):
    """
    # Parameters

    value : `float`
        The value to average.
    """
    _total_value = list(self.detach_tensors(value))[0]
    _count = 1
    if is_distributed():
        device = torch.device("cpu")
        count = torch.tensor(_count).to(device)
        total_value = torch.tensor(_total_value).to(device)
        dist.all_reduce(count, op=dist.ReduceOp.SUM)
        dist.all_reduce(total_value, op=dist.ReduceOp.SUM)
        _count = count.item()
        _total_value = total_value.item()
    self._count += _count
    self._total_value += _total_value


# Traceback (RuntimeError)
0 | 2020-09-03 12:39:41,240 - INFO - allennlp.training.trainer - Beginning training.
0 | 2020-09-03 12:39:41,240 - INFO - allennlp.training.trainer - Epoch 0/9
0 | 2020-09-03 12:39:41,241 - INFO - allennlp.training.trainer - Worker 0 memory usage MB: 2984.032
0 | 2020-09-03 12:39:41,241 - INFO - allennlp.training.trainer - Worker 1 memory usage MB: 2985.344
0 | 2020-09-03 12:39:41,241 - INFO - allennlp.training.trainer - Worker 2 memory usage MB: 2988.028
0 | 2020-09-03 12:39:41,241 - INFO - allennlp.training.trainer - Worker 3 memory usage MB: 2984.496
0 | 2020-09-03 12:39:41,241 - INFO - allennlp.training.trainer - Worker 4 memory usage MB: 2989.064
0 | 2020-09-03 12:39:41,241 - INFO - allennlp.training.trainer - Worker 5 memory usage MB: 2987.996
0 | 2020-09-03 12:39:41,241 - INFO - allennlp.training.trainer - Worker 6 memory usage MB: 2988.592
0 | 2020-09-03 12:39:41,241 - INFO - allennlp.training.trainer - Worker 7 memory usage MB: 2991.332
reading instances: 0it [00:00, ?it/s]0 | 2020-09-03 12:39:41,601 - INFO - allennlp.training.trainer - GPU 0 memory usage MB: 1614
0 | 2020-09-03 12:39:41,601 - INFO - allennlp.training.trainer - GPU 1 memory usage MB: 1614
0 | 2020-09-03 12:39:41,601 - INFO - allennlp.training.trainer - GPU 2 memory usage MB: 1614
0 | 2020-09-03 12:39:41,601 - INFO - allennlp.training.trainer - GPU 3 memory usage MB: 1614
0 | 2020-09-03 12:39:41,601 - INFO - allennlp.training.trainer - GPU 4 memory usage MB: 1614
0 | 2020-09-03 12:39:41,601 - INFO - allennlp.training.trainer - GPU 5 memory usage MB: 1614
0 | 2020-09-03 12:39:41,601 - INFO - allennlp.training.trainer - GPU 6 memory usage MB: 1614
0 | 2020-09-03 12:39:41,601 - INFO - allennlp.training.trainer - GPU 7 memory usage MB: 1614
reading instances: 0it [00:00, ?it/s]0 | 2020-09-03 12:39:41,603 - INFO - allennlp.training.trainer - Training
reading instances: 0it [00:00, ?it/s]0 | 2020-09-03 12:39:41,605 - INFO - src.models.language_model_conll_reader - Loading data from data/processed/no_bi_tags/new_model1_train.txt
reading instances: 0it [00:00, ?it/s]/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/metrics/average.py:36: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
total_value = torch.tensor(_total_value).to(device)
reading instances: 63it [00:00, 91.52it/s]
/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/metrics/average.py:36: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
total_value = torch.tensor(_total_value).to(device)
reading instances: 63it [00:00, 85.48it/s]
/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/metrics/average.py:36: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
total_value = torch.tensor(_total_value).to(device)
0it [00:00, ?it/s]
reading instances: 63it [00:00, 83.76it/s]
/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/metrics/average.py:36: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
total_value = torch.tensor(_total_value).to(device)
reading instances: 63it [00:00, 83.20it/s]
/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/metrics/average.py:36: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
total_value = torch.tensor(_total_value).to(device)
reading instances: 63it [00:00, 81.84it/s]
/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/metrics/average.py:36: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
total_value = torch.tensor(_total_value).to(device)
/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/metrics/average.py:36: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
total_value = torch.tensor(_total_value).to(device)
reading instances: 63it [00:00, 79.57it/s]
reading instances: 63it [00:00, 80.02it/s]
/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/metrics/average.py:36: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
total_value = torch.tensor(_total_value).to(device)
reading instances: 63it [00:00, 79.14it/s]
Traceback (most recent call last):
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/bin/allennlp", line 8, in <module>
sys.exit(run())
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/__main__.py", line 34, in run
main(prog="allennlp")
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/commands/__init__.py", line 92, in main
args.func(args)
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/commands/train.py", line 118, in train_model_from_args
file_friendly_logging=args.file_friendly_logging,
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/commands/train.py", line 177, in train_model_from_file
file_friendly_logging=file_friendly_logging,
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/commands/train.py", line 304, in train_model
nprocs=num_procs,
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 200, in spawn
return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 158, in start_processes
while not context.join():
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 119, in join
raise Exception(msg)
Exception:

-- Process 2 terminated with the following error:
Traceback (most recent call last):
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 20, in _wrap
fn(i, *args)
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/commands/train.py", line 439, in _train_worker
metrics = train_loop.run()
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/commands/train.py", line 501, in run
return self.trainer.train()
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/trainer.py", line 867, in train
train_metrics = self._train_epoch(epoch)
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/trainer.py", line 589, in _train_epoch
batch_outputs = self.batch_outputs(batch, for_training=True)
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/trainer.py", line 479, in batch_outputs
output_dict = self._pytorch_model(**batch)
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
result = self.forward(*input, **kwargs)
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/torch/nn/parallel/distributed.py", line 511, in forward
output = self.module(*inputs[0], **kwargs[0])
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
result = self.forward(*input, **kwargs)
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp_models/lm/models/language_model.py", line 297, in forward
self._perplexity(average_loss)
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/allennlp/training/metrics/average.py", line 37, in __call__
dist.all_reduce(count, op=dist.ReduceOp.SUM)
File "/app/local/anaconda3/envs/torch1.6_allennlp1.1/lib/python3.7/site-packages/torch/distributed/distributed_c10d.py", line 936, in all_reduce
work = _default_pg.allreduce([tensor], opts)
RuntimeError: Tensors must be CUDA and dense

# Fixed code
def __call__(self, value):
    """
    # Parameters

    value : `float`
        The value to average.
    """
    _total_value = list(self.detach_tensors(value))[0]
    _count = 1
    if is_distributed():
        device = torch.device("cuda" if dist.get_backend() == "nccl" else "cpu")
        count = torch.tensor(_count).to(device)
        total_value = torch.tensor(_total_value).to(device)
        dist.all_reduce(count, op=dist.ReduceOp.SUM)
        dist.all_reduce(total_value, op=dist.ReduceOp.SUM)
        _count = count.item()
        _total_value = total_value.item()
    self._count += _count
    self._total_value += _total_value