DistributedDataParallel does not reduce the training time

Smith_Jack: <p>Hello, I am using  DistributedDataParallel to parallel my model, however, I find that it does not reduce the training time as I increase the number of processes.<br>
When I run the code with</p>
<pre data-code-wrap="bash"><code class="lang-bash">time torchrun --nnodes=1 --nproces_per_node=1 ./test.py 
</code></pre>
<p>and</p>
<pre data-code-wrap="bash"><code class="lang-bash">time torchrun --nnodes=1 --nproces_per_node=4 ./test.py 
</code></pre>
<p>The real times are both 53s. Can anyone help me to check where I did wrong ? Thanks.<br>
Here is my code</p>
<pre data-code-wrap="python"><code class="lang-python">import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data import DistributedSampler


dist.init_process_group("gloo")

batch_size = 64
drop_last = True 
shuffle = True

training_data = datasets.FashionMNIST(
    root="data",
    train=True,
    download=True,
    transform=ToTensor()
)

test_data = datasets.FashionMNIST(
    root="data",
    train=False,
    download=True,
    transform=ToTensor()
)


rank = dist.get_rank()
world_size = dist.get_world_size()
train_sampler = DistributedSampler(training_data, num_replicas=world_size, rank=rank, drop_last = drop_last, shuffle=shuffle)
test_sampler = DistributedSampler(test_data, num_replicas=world_size, rank=rank, drop_last = drop_last, shuffle=shuffle)

train_dataloader = DataLoader(training_data, batch_size=batch_size, sampler=train_sampler)
test_dataloader = DataLoader(test_data, batch_size=batch_size, sampler=test_sampler)


class NeuralNetwork(nn.Module):
    def __init__(self):
        super().__init__()
        self.flatten = nn.Flatten()
        self.linear_relu_stack = nn.Sequential(
            nn.Linear(28*28, 512),
            nn.ReLU(),
            nn.Linear(512, 512),
            nn.ReLU(),
            nn.Linear(512, 10),
        )

    def forward(self, x):
        x = self.flatten(x)
        logits = self.linear_relu_stack(x)
        return logits

def train_loop(iepoch, dataloader, model, loss_fn, optimizer):
    dataloader.sampler.set_epoch(iepoch) 
    size = len(dataloader.dataset)
    # Set the model to training mode - important for batch normalization and dropout layers
    # Unnecessary in this situation but added for best practices
    model.train()
    for batch, (X, y) in enumerate(dataloader):
        # Compute prediction and loss
        pred = model(X)
        loss = loss_fn(pred, y)

        # Backpropagation
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()

        if batch % 100 == 0:
            loss, current = loss.item(), batch * batch_size + len(X)
            print(f"loss: {loss:&gt;7f}  [{current:&gt;5d}/{size:&gt;5d}]")


def test_loop(iepoch, dataloader, model, loss_fn):
    dataloader.sampler.set_epoch(iepoch) 
    # Set the model to evaluation mode - important for batch normalization and dropout layers
    # Unnecessary in this situation but added for best practices
    model.eval()
    size = len(dataloader.dataset)
    num_batches = len(dataloader)
    test_loss, correct = 0, 0

    # Evaluating the model with torch.no_grad() ensures that no gradients are computed during test mode
    # also serves to reduce unnecessary gradient computations and memory usage for tensors with requires_grad=True
    with torch.no_grad():
        for X, y in dataloader:
            pred = model(X)
            test_loss += loss_fn(pred, y).item()
            correct += (pred.argmax(1) == y).type(torch.float).sum().item()

    print("rank = ", dist.get_rank(), "num_batches = ", num_batches, "size = ", size, "test_loss = ", test_loss, "correct = ", correct)
    test_loss /= num_batches
    correct /= size
    print(f"Test Error: \n Accuracy: {(100*correct):&gt;0.1f}%, Avg loss: {test_loss:&gt;8f} \n")



device = torch.device("cpu")
model = NeuralNetwork().to(device=device)

model = DDP(model)
learning_rate = 1e-3
epochs = 5
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

for iepoch in range(epochs):
    print(f"Epoch {iepoch+1}\n-------------------------------")
    train_loop(iepoch, train_dataloader, model, loss_fn, optimizer)
    test_loop(iepoch, test_dataloader, model, loss_fn)
print("Done!")

dist.destroy_process_group()
</code></pre>

yenchenlin: <p>When you run</p>
<pre><code class="lang-auto">time torchrun --nnodes=1 --nproces_per_node=4 ./test.py 
</code></pre>
<p>you expand the batch size by 4 times.</p>
<p>DistributedDataParallel helps you go through 4 times more data in the same amount of time. This reduces the total time you need to go through the whole dataset once (an epoch).</p>

Smith_Jack: <p>Thanks for your kind reply, then how should I modify my code ? Should I change</p>
<aside class="quote no-group" data-username="Smith_Jack" data-post="1" data-topic="204975">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/smith_jack/48/70752_2.png" class="avatar"> Smith_Jack:</div>
<blockquote>
<p><code>train_sampler = DistributedSampler(training_data, num_replicas=world_size, rank=rank, drop_last = drop_last, shuffle=shuffle)</code></p>
</blockquote>
</aside>
<p>to (just delete the <strong>num_replicas</strong> parameter)<br>
<code>train_sampler = DistributedSampler(training_data, rank=rank, drop_last = drop_last, shuffle=shuffle)</code><br>
I haved checked the new code. When I changed epochs to 50, <code>2m56.474s</code> was used if <code>nproc_per_node=4</code>, and <code>5m45.610s</code> is used if <code>nproc_per_node=1</code>. Is this indicated that DDP worked correctly ?</p>
<p>May I ask you some other questions? The pytorch docs and some other tutorials make me rather confused.</p>
<ol>
<li>Did I use torchrun correctly? I run the code on multi-core <strong>CPU</strong>.<br>
<code>torchrun --nnodes=1 --nproc_per_node=4 ./test.py</code><br>
In some tutorials, a keyword <code>--standalone</code> is added. But I get segmentation fault erros when I use this keyword.</li>
</ol>
<pre data-code-wrap="bash"><code class="lang-bash">Fatal Python error: Segmentation fault

Current thread 0x00007fc2ccbbf740 (most recent call first):
  File "/mnt/d/software_install/debian/pytorch/lib/python3.12/site-packages/torch/distributed/elastic/rendezvous/c10d_rendezvous_backend.py", line 113 in _call_store
  File "/mnt/d/software_install/debian/pytorch/lib/python3.12/site-packages/torch/distributed/elastic/rendezvous/c10d_rendezvous_backend.py", line 64 in __init__
  File "/mnt/d/software_install/debian/pytorch/lib/python3.12/site-packages/torch/distributed/elastic/rendezvous/c10d_rendezvous_backend.py", line 259 in create_backend
  File "/mnt/d/software_install/debian/pytorch/lib/python3.12/site-packages/torch/distributed/elastic/rendezvous/registry.py", line 36 in _create_c10d_handler
  File "/mnt/d/software_install/debian/pytorch/lib/python3.12/site-packages/torch/distributed/elastic/rendezvous/api.py", line 263 in create_handler
  File "/mnt/d/software_install/debian/pytorch/lib/python3.12/site-packages/torch/distributed/elastic/rendezvous/registry.py", line 66 in get_rendezvous_handler
  File "/mnt/d/software_install/debian/pytorch/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 235 in launch_agent
  File "/mnt/d/software_install/debian/pytorch/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 132 in __call__
  File "/mnt/d/software_install/debian/pytorch/lib/python3.12/site-packages/torch/distributed/run.py", line 870 in run
  File "/mnt/d/software_install/debian/pytorch/lib/python3.12/site-packages/torch/distributed/run.py", line 879 in main
  File "/mnt/d/software_install/debian/pytorch/lib/python3.12/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 347 in wrapper
  File "/mnt/d/software_install/debian/pytorch/bin/torchrun", line 33 in &lt;module&gt;

Extension modules: mkl._mklinit, mkl._py_mkl_service, numpy.core._multiarray_umath, numpy.core._multiarray_tests, numpy.linalg._umath_linalg, numpy.fft._pocketfft_internal, numpy.random._common, numpy.random.bit_generator, numpy.random._bounded_integers, numpy.random._mt19937, numpy.random.mtrand, numpy.random._philox, numpy.random._pcg64, numpy.random._sfc64, numpy.random._generator, torch._C, torch._C._fft, torch._C._linalg, torch._C._nested, torch._C._nn, torch._C._sparse, torch._C._special (total: 22)
Segmentation fault
</code></pre>
<ol start="2">
<li>Should the <strong>torch.multiprocessing.spawn</strong> be used ? Again, some of the tutorials use it but others don’t. (And someone told me that <strong>torch.multiprocessing</strong> is a low level module. It is not recommended.)</li>
<li>In most tutorials I can find some codes like this</li>
</ol>
<pre data-code-wrap="python"><code class="lang-python">dist.init_process_group("gloo", rank=rank, world_size=world_size)
model = ToyModel().to(rank)
ddp_model = DDP(model, device_ids=[rank])
</code></pre>
<p>When I run the code on multi-core CPU, I should change them to</p>
<pre data-code-wrap="python"><code class="lang-python">dist.init_process_group("gloo")
model = ToyModel()
ddp_model = DDP(mode)
</code></pre>
<p>Is this right ?<br>
Thanks again if you can help me with these questions.</p>

yenchenlin: <p>To see the effect of faster iteration you just need to divide the batch size of your dataloader by 4 when you run</p>
<pre><code class="lang-auto">time torchrun --nnodes=1 --nproces_per_node=4 ./test.py 
</code></pre>
<p>In this case, each GPU will receive 1/4 of the data it originally received and hence reduce the time per iteration.</p>

Smith_Jack: <p>Yes, you are right. Even I don’t set num_replicas, it is set to be the number of process automatically. Thank you.</p>

Smith_Jack: <p>I have divide the batch_size by 4, however, the time consumption is still larger than only one process <img src="https://discuss.pytorch.org/images/emoji/apple/smiling_face_with_tear.png?v=12" title=":smiling_face_with_tear:" class="emoji" alt=":smiling_face_with_tear:" loading="lazy" width="20" height="20"> . In this post, I said the  time consumption is shorter, however, that’s because the true batch_size is 4 times larger in <code>nproc_per_node=4</code>  than <code>nproc_per_node=1</code>. In the code below, I print a variable <code>uu</code>, it is 938 in both cases, so the number of iteration is the same.</p>
<aside class="quote no-group" data-username="Smith_Jack" data-post="3" data-topic="204975">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/smith_jack/48/70752_2.png" class="avatar"> Smith_Jack:</div>
<blockquote>
<p>I haved checked the new code. When I changed epochs to 50, <code>2m56.474s</code> was used if <code>nproc_per_node=4</code>, and <code>5m45.610s</code> is used if <code>nproc_per_node=1</code>. Is this indicated that DDP worked correctly ?</p>
</blockquote>
</aside>
<p>Is this because I run my code on CPU ? (I have no GPU)</p>
<pre data-code-wrap="python"><code class="lang-python">import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data import DistributedSampler
dist.init_process_group("gloo")

rank = dist.get_rank()
world_size = dist.get_world_size()

tot_batch_size = 64
batch_size = int(tot_batch_size/world_size)
drop_last = True 
shuffle = True

training_data = datasets.FashionMNIST(
    root="data",
    train=True,
    download=True,
    transform=ToTensor()
)
train_sampler = DistributedSampler(training_data, rank=rank, drop_last = drop_last, shuffle=shuffle)
train_dataloader = DataLoader(training_data, batch_size=batch_size, sampler=train_sampler)

class NeuralNetwork(nn.Module):
    def __init__(self):
        super().__init__()
        self.flatten = nn.Flatten()
        self.linear_relu_stack = nn.Sequential(
            nn.Linear(28*28, 512),
            nn.ReLU(),
            nn.Linear(512, 512),
            nn.ReLU(),
            nn.Linear(512, 10),
        )

    def forward(self, x):
        x = self.flatten(x)
        logits = self.linear_relu_stack(x)
        return logits

def train_loop(iepoch, dataloader, model, loss_fn, optimizer):
    dataloader.sampler.set_epoch(iepoch) 
    size = len(dataloader.dataset)
    # Set the model to training mode - important for batch normalization and dropout layers
    # Unnecessary in this situation but added for best practices
    model.train()
    uu = 0
    for batch, (X, y) in enumerate(dataloader):
        uu += 1
        # Compute prediction and loss
        pred = model(X)
        loss = loss_fn(pred, y)

        # Backpropagation
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()

        if batch % 100 == 0:
            loss, current = loss.item(), batch * batch_size + len(X)
            if rank == 0:
                print(f"loss: {loss:&gt;7f}  [{current:&gt;5d}/{size:&gt;5d}]")
    if rank == 0:
        print(uu)

device = torch.device("cpu")
model = NeuralNetwork().to(device=device)
model = DDP(model)
learning_rate = 1e-3
epochs = 5
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

for iepoch in range(epochs):
    if rank == 0:
        print(f"Epoch {iepoch+1}\n-------------------------------")
    train_loop(iepoch, train_dataloader, model, loss_fn, optimizer)

dist.destroy_process_group()
</code></pre>