Issue 1:
I'm trying distributed training with pytorch, and I encountered the following error.

File "/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/nn/modules/module.py", line 259, in __call__
result = self.forward(*input, **kwargs)
File "/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/nn/parallel/distributed.py", line 154, in forward
self._sync_params()
File "/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/nn/parallel/distributed.py", line 183, in _sync_params
flat_buffers = _flatten_tensors(buffers)
File "/home.local/kamo/.anyenv/envs/pyenv/versions/torch2/lib/python3.6/site-packages/torch/_utils.py", line 111, in _flatten_tensors
flat = tensors[0].new(size)
IndexError: list index out of range

Reproduce code:

# main.py
import os

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable
import torch.distributed as dist
import torch.utils.data.distributed

class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
self.conv2_drop = nn.Dropout2d()
self.fc1 = nn.Linear(320, 50)
self.fc2 = nn.Linear(50, 10)

def forward(self, x):
x = F.relu(F.max_pool2d(self.conv1(x), 2))
x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
x = x.view(-1, 320)
x = F.relu(self.fc1(x))
x = F.dropout(x, training=self.training)
x = self.fc2(x)
return F.log_softmax(x)

def main():
BACKEND = os.environ['BACKEND']
WORLD_SIZE = os.environ['WORLD_SIZE']
INIT_METHOD = os.getenv('INIT_METHOD', 'env://')

dist.init_process_group(
init_method=INIT_METHOD,
backend=BACKEND, world_size=int(WORLD_SIZE))
train_dataset = datasets.MNIST(root='./torch', train=True, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,),
(0.3081,))]))
train_sampler =\
torch.utils.data.distributed.DistributedSampler(train_dataset)
train_loader = torch.utils.data.DataLoader(
train_dataset, sampler=train_sampler,
batch_size=64, shuffle=False, num_workers=0, pin_memory=False)

model = Net()
model.cuda()
model = torch.nn.parallel.DistributedDataParallel(model)
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)

train_sampler.set_epoch(0)
model.train()
for data, target in train_loader:
data, target = data.cuda(), target.cuda()
data, target = Variable(data), Variable(target)
optimizer.zero_grad()
output = model(data)
loss = F.nll_loss(output, target)
loss.backward()
optimizer.step()

if __name__ == '__main__':
import argparse
p = argparse.ArgumentParser()
main()

Launching shell script:

#!/bin/sh

distributed_set_up() {
export TEMP_DIR="$(mktemp -d)"
rm -rf "$TEMP_DIR/"*
mkdir "$TEMP_DIR/barrier"
mkdir "$TEMP_DIR/test_dir"
}

distributed_tear_down() {
rm -rf "$TEMP_DIR"
}

distributed_set_up
ls $TEMP_DIR
backend=gloo
world_size=2
for i in seq 0 $(expr $world_size - 1);do
MASTER_ADDR='29500' MASTER_PORT='127.0.0.1' RANK=$i BACKEND=$backend WORLD_SIZE=$world_size INIT_METHOD='file://'$TEMP_DIR'/shared_init_file' python main.py &
done
wait
distributed_tear_down

Perhaps you are not taking account modules not having any buffers, so I modified https://github.com/pytorch/pytorch/blob/master/torch/nn/parallel/distributed.py#L182-L192 like this and it works,

buffers = list(self.module._all_buffers())
if len(buffers) > 0:
flat_buffers = _flatten_tensors(buffers)
dist.broadcast(flat_buffers, 0)
for buf, synced in zip(buffers, _unflatten_tensors(flat_buffers, buffers)):
buf.copy_(synced)

# intra-node buffer sync
result = broadcast_coalesced(buffers, self.device_ids, self.broadcast_bucket_size)
for tensors, module in zip(result[1:], self._module_copies[1:]):
for tensor, buf in zip(tensors, module._all_buffers()):
buf.set_(tensor)

My correction is not right?

After this, maybe this is another problem, but i got the following message at process shutdown. The training seems to success, so I think this is not a critical problem.

terminate called after throwing an instance of 'gloo::EnforceNotMet'
what(): [enforce fail at /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249] error == cudaSuccess. 29 vs 0. Error at: /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249: driver shutting down
terminate called after throwing an instance of 'gloo::EnforceNotMet'
what(): [enforce fail at /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249] error == cudaSuccess. 29 vs 0. Error at: /home.local/kamo/work/pytorch/torch/lib/gloo/gloo/cuda.cu:249: driver shutting down
./run.sh: line 22: 19545 Aborted (core dumped) MASTER_ADDR='29500' MASTER_PORT='127.0.0.1' RANK=$i BACKEND=$backend WORLD_SIZE=$world_size INIT_METHOD='file://'$TEMP_DIR'/shared_init_file' python main.py
./run.sh: line 22: 19546 Aborted (core dumped) MASTER_ADDR='29500' MASTER_PORT='127.0.0.1' RANK=$i BACKEND=$backend WORLD_SIZE=$world_size INIT_METHOD='file://'$TEMP_DIR'/shared_init_file' python main.py

My environment:

OS: CentOS7
gcc: 4.8.5 20150623 (Red Hat 4.8.5-11) (GCC)
Python3.6
Pytorch: de757805fcdbcbb831d51827d72e73e55a49a106 (built by setup.py)
GPU: GeForce GTX 1080
CUDA8.0
CUDNN6.0
Without MPI

Issue 2:
🐛 Bug

To Reproduce

Steps to reproduce the behavior:

m = nn.MultiheadAttention(embed_dim=20, num_heads=5, dropout=0.1)

i = torch.randn(4, 5, 20)
q = k = v = i.transpose(0, 1)
key_padding_mask = seq_len_to_mask([5, 3, 1, 0], max_len=5)

ao, aow = m(q, k, v, key_padding_mask=key_padding_mask)
ao = ao.transpose(0, 1)

print(aow)

the print is:

tensor([[[0.2822, 0.2024, 0.1926, 0.2565, 0.1463],
[0.2020, 0.1906, 0.1721, 0.2159, 0.2425],
[0.2074, 0.2127, 0.0882, 0.2422, 0.3333],
[0.2298, 0.1938, 0.2093, 0.1432, 0.2335],
[0.2531, 0.1772, 0.3045, 0.1246, 0.1881]],

[[0.3693, 0.3857, 0.3561, 0.0000, 0.0000],
[0.3655, 0.2712, 0.3918, 0.0000, 0.0000],
[0.2230, 0.3795, 0.2549, 0.0000, 0.0000],
[0.1698, 0.3805, 0.2318, 0.0000, 0.0000],
[0.2986, 0.3093, 0.4405, 0.0000, 0.0000]],

[[0.8889, 0.0000, 0.0000, 0.0000, 0.0000],
[1.1111, 0.0000, 0.0000, 0.0000, 0.0000],
[1.1111, 0.0000, 0.0000, 0.0000, 0.0000],
[1.1111, 0.0000, 0.0000, 0.0000, 0.0000],
[1.1111, 0.0000, 0.0000, 0.0000, 0.0000]],

[[ nan, nan, nan, nan, nan],
[ nan, nan, nan, nan, nan],
[ nan, nan, nan, nan, nan],
[ nan, nan, nan, nan, nan],
[ nan, nan, nan, nan, nan]]], grad_fn=<DivBackward0>)

Expected behavior

Environment

Please copy and paste the output from our

environment collection script

(or fill out the checklist below manually).

You can get the script and run it with:

wget https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py
# For security purposes, please check the contents of collect_env.py before running it.
python collect_env.py

PyTorch Version (e.g., 1.0): 1.3.0

OS (e.g., Linux): macos

How you installed PyTorch (conda, pip, source):

Build command you used (if compiling from source):

Python version:

CUDA/cuDNN version:

GPU models and configuration:

Any other relevant information:

Additional context

cc @ezyang @gchanan @zou3519