Does DDP make computations non-deterministic？

username: yanz0920
cooked: <p>When I training the model several times by single process，the resulting models are always consistent.<br>
But, when I use DDP to train the model, the modle is different every time. This makes me very confused！</p>
<p>Here  are the codes for training：</p>
<p>#!/usr/bin/env python</p>
<p>import sys<br>
import os<br>
import numpy as np<br>
import random<br>
import torch<br>
import torch.nn as nn<br>
import torch.nn.functional as F<br>
import torch.optim as optim<br>
from torchvision import datasets, transforms<br>
from sampler import DistributedSampler</p>
<p>class CNN(nn.Module):<br>
def <strong>init</strong>(self):<br>
super(CNN, self).<strong>init</strong>()<br>
self.conv1 = nn.Conv2d(3, 64, 5, padding=1)<br>
self.conv2 = nn.Conv2d(64, 64, 3, padding=1)<br>
self.pool = nn.MaxPool2d(2, 2)<br>
self.conv3 = nn.Conv2d(64, 128, 3, padding=1)<br>
self.conv4 = nn.Conv2d(128, 128, 3, padding=1)<br>
self.fc1 = nn.Linear(128 * 7 * 7, 1024)<br>
self.fc2 = nn.Linear(1024, 10)</p>
<p>def forward(self, x):<br>
x = F.relu(self.conv1(x))<br>
x = F.relu(self.pool(self.conv2(x)))<br>
x = F.relu(self.conv3(x))<br>
x = F.relu(self.pool(self.conv4(x)))<br>
x = x.view(-1, 128 * 7 * 7)<br>
x = F.relu(self.fc1(x))<br>
x = self.fc2(x)<br>
return x</p>
<p>os.environ[“NCCL_IB_DISABLE”] = ‘1’<br>
init_method = sys.argv[1]<br>
world_size = int(sys.argv[2])<br>
rank = int(sys.argv[3])<br>
torch.distributed.init_process_group(<br>
backend=“nccl”,<br>
init_method=init_method,<br>
world_size=world_size,<br>
rank=rank-1)</p>
<p>train_data = datasets.CIFAR10(root=’./data’, train=True, download=True, transform=transforms.ToTensor())<br>
train_sampler = torch.utils.data.distributed.DistributedSampler(train_data, num_replicas=world_size, rank=rank-1)<br>
train_loader = torch.utils.data.DataLoader(train_data, batch_size=64, pin_memory=True,<br>
shuffle=False, num_workers=0, sampler=train_sampler,<br>
worker_init_fn=lambda x: np.random.seed(1))<br>
test_data = datasets.CIFAR10(root=’./data’, train=False, download=True, transform=transforms.ToTensor())<br>
test_sampler = torch.utils.data.distributed.DistributedSampler(test_data, num_replicas=world_size, rank=rank-1)<br>
test_loader = torch.utils.data.DataLoader(test_data, batch_size=64, shuffle=False, num_workers=0, sampler=test_sampler)</p>
<p>torch.manual_seed(2)<br>
torch.cuda.manual_seed_all(2)<br>
torch.backends.cudnn.deterministic = True<br>
torch.backends.cudnn.benchmark = False<br>
np.random.seed(1)<br>
random.seed(1)</p>
<p>for i in range(torch.cuda.device_count()):<br>
device = torch.device(“cuda:” + str(i))<br>
x = torch.zeros(1)<br>
try:<br>
x = x.to(device)<br>
break<br>
except RuntimeError:<br>
pass</p>
<p>torch.cuda.set_device(device)<br>
model = CNN()<br>
model.to(device)<br>
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device])</p>
<p>criterion = nn.CrossEntropyLoss()<br>
optimizer = optim.Adam(model.parameters(), lr=0.001)<br>
model.train()<br>
for epoch in range(2):<br>
iter = 0<br>
for inputs, labels in train_loader:<br>
inputs, labels = inputs.to(device), labels.to(device)<br>
optimizer.zero_grad()<br>
outputs = model(inputs)<br>
loss = criterion(outputs, labels)<br>
loss.backward()<br>
optimizer.step()<br>
iter += 1<br>
print(“Epoch=%d  iter=%d  loss=%.4f” % (epoch, iter, loss.item()))</p>
<p>for name, para in model.named_parameters():<br>
text = name + “:”<br>
para = para.view(-1)<br>
for i in range(min(para.size(0), 32)):<br>
text += " %.10f" % para[i].item()<br>
print(text)</p>
<p>correct, tot = 0, 0<br>
model.eval()<br>
with torch.no_grad():<br>
for inputs, labels in test_loader:<br>
inputs, labels = inputs.to(device), labels.to(device)<br>
outputs = model(inputs)<br>
_, predicted = torch.max(outputs.data, 1)<br>
tot += labels.size(0)<br>
correct += (predicted == labels).sum().item()</p>
<p>print(correct / tot * 100)</p>

username: yanz0920
cooked: <p>Finally，I find that the randomness caused by NCCL rings. Fix the rings and it becomes deterministic.</p>

username: lezasantaizi
cooked: <p>how to fix the rings ?</p>