Horovod using only one gpu instead of all avaialable

Raj_Roy: <p>Hi, I am new to pytorch and I am facing issues when I am trying to run multigpu using Horovod. Even torch.cude.device.count() is 6 but it is using only one gpu and if I print local rank I get 0. I would be very grateful for your help.<br>
I use this command for submitting the job <img src="https://discuss.pytorch.org/images/emoji/apple/slight_smile.png?v=9" title=":slight_smile:" class="emoji" alt=":slight_smile:"><br>
jsrun -bpacked:7 -g6 -a1 -c42 -r1 python ./pytorch_code.py</p>
<p>Brief version of my code</p>
<p>import math<br>
import os<br>
import sys<br>
import numpy as np</p>
<h1>
<a name="read-all-these-from-a-file-1" class="anchor" href="#read-all-these-from-a-file-1"></a>READ ALL THESE FROM A FILE</h1>
<p>import rough_copy<br>
import torch.optim as optim</p>
<p>DECAY_UPDATE_INTERVAL = 25<br>
BATCH_SIZE = 2</p>
<h3>
<a name="horovod-2" class="anchor" href="#horovod-2"></a>HOROVOD</h3>
<p>print("HVD AREA ")<br>
backend = ‘horovod’<br>
world_size = int(os.environ[‘OMPI_COMM_WORLD_SIZE’])<br>
world_rank = int(os.environ[‘OMPI_COMM_WORLD_RANK’])<br>
local_rank = int(os.environ[‘OMPI_COMM_WORLD_LOCAL_RANK’])</p>
<h1>
<a name="horovod-and-ddl-share-the-same-usage-mode-3" class="anchor" href="#horovod-and-ddl-share-the-same-usage-mode-3"></a>Horovod and DDL share the same usage mode</h1>
<p>if backend in [‘horovod’,‘ddl’]:<br>
print(“using hvd”)<br>
import horovod.torch as hvd<br>
hvd.init()<br>
torch.cuda.set_device(local_rank)<br>
print("local rank "+str(local_rank))<br>
else:<br>
import torch.distributed as dist<br>
# Initialze NCCL backend with envrionment variables<br>
if backend == ‘nccl’:<br>
import subprocess<br>
get_master = “echo $(cat {} | sort | uniq | grep -v batch | grep -v login | head -1)”.format(os.environ[‘LSB_DJOB_HOSTFILE’])<br>
os.environ[‘MASTER_ADDR’] = str(subprocess.check_output(get_master, shell=True))[2:-3]<br>
os.environ[‘MASTER_PORT’] = “23456”<br>
os.environ[‘WORLD_SIZE’] = os.environ[‘OMPI_COMM_WORLD_SIZE’]<br>
os.environ[‘RANK’] = os.environ[‘OMPI_COMM_WORLD_RANK’]<br>
# NCCL and MPI initialization<br>
print(“using ncll”)<br>
dist.init_process_group(backend,<br>
rank=world_rank, world_size=world_size)<br>
print(“NONE”)<br>
print("device ss "+str(torch.cuda.device_count()))</p>
<p>class my_dataset(data.Dataset):<br>
def initialize(self, _feat_path, _tr_ros, _ss, _intra, _label_path, _file_list, _max_len):<br>
.<br>
.</p>
<pre><code>def __getitem__(self, index):
    .
    .
    .
def __len__(self):
    return self.size
</code></pre>
<p>nThreads =6</p>
<p>train_file_list = text_file_reader(train_file)<br>
train_dataset = my_dataset()<br>
train_dataset.initialize(_feat_path=feature_path, _label_path=label_path, _ss=ss_path, _intra=intra_path,<br>
_tr_ros=tr_path,<br>
_file_list=train_file_list, _max_len=MAX_LENGTH)</p>
<p>train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=nThreads)</p>
<p>val_file_list = text_file_reader(val_file)<br>
val_dataset = my_dataset()<br>
val_dataset.initialize(_feat_path=feature_path, _label_path=label_path, _ss=ss_path, _intra=intra_path, _tr_ros=tr_path,<br>
_file_list=val_file_list, _max_len=MAX_LENGTH)</p>
<p>val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=nThreads)</p>
<p>print("MODELLOADER AREA ")<br>
input = torch.randn((1, FEATURES, MAX_LENGTH, MAX_LENGTH)).cuda()<br>
import rough_copy</p>
<p>model = rough_copy.ResNet_custom(img_channel=FEATURES, num_classes=MAX_LENGTH * MAX_LENGTH, _depth=RESNET_DEPTH).cuda()<br>
output = model(input)</p>
<p>criterion = nn.BCELoss()<br>
optimizer = optim.Adam(model.parameters(), lr=0.0001, weight_decay=0)<br>
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=DECAY_UPDATE_INTERVAL, gamma=0.5)<br>
print("HVD  AREA ")</p>
<p>if backend in [‘horovod’,‘ddl’]:<br>
# Horovod: wrap optimizer with DistributedOptimizer.<br>
optimizer = hvd.DistributedOptimizer(optimizer,<br>
named_parameters=model.named_parameters())<br>
# Horovod: broadcast parameters &amp; optimizer state.<br>
hvd.broadcast_parameters(model.state_dict(), root_rank=0)<br>
hvd.broadcast_optimizer_state(optimizer, root_rank=0)<br>
else:<br>
def average_gradients(model):<br>
“”" Gradient averaging. “”"<br>
for param in model.parameters():<br>
dist.all_reduce(param.grad.data, op=dist.ReduceOp.SUM)<br>
param.grad.data /= world_size</p>
<p>print("Restart AREA ")</p>
<p>Current_EPOCH = 0<br>
weights = list(filter(os.path.isfile, glob.glob(HISTORY_FILE + “*”)))<br>
weights.sort(key=lambda x: os.path.getmtime(x))<br>
if len(weights) &gt; 0:<br>
checkpoint=torch.load(weights[len(weights)-1])<br>
model.load_state_dict(checkpoint[‘model_state_dict’])<br>
optimizer.load_state_dict(checkpoint[‘optimizer_state_dict’])<br>
scheduler.load_state_dict(checkpoint[‘scheduler_state_dict’])<br>
epoch = checkpoint[‘epoch’]<br>
Current_EPOCH =epoch+1<br>
model.train()</p>
<p>print("EPOCH AREA ")<br>
for epoch in range(Current_EPOCH, EPOCH):<br>
model.train()<br>
print("TRAINGNG AREA ")<br>
for i, data in enumerate(train_dataloader):<br>
features = data[‘feat’].float().cuda()<br>
label = data[‘ground_truth’].float().cuda()<br>
optimizer.zero_grad()<br>
output_final = torch.squeeze(model(features))<br>
loss = criterion(torch.squeeze(output_val), torch.squeeze(label))<br>
print(loss)<br>
running_train_loss += loss.item()<br>
loss.backward()<br>
optimizer.step()<br>
scheduler.step()</p>
<pre><code>torch.save({
    'epoch': epoch,
    'model_state_dict': model.state_dict(),
    'optimizer_state_dict': optimizer.state_dict(),
    'scheduler_state_dict': scheduler.state_dict()
}, HISTORY_FILE + "/weighths_" + str(epoch))

print("VALIDATIOn AREA ")
with torch.no_grad():
    "some custom codes"
Current_EPOCH = Current_EPOCH + 1
</code></pre>
<p><strong>Output of horovodrun --check-build</strong></p>
<p>Available Frameworks:<br>
[X] TensorFlow<br>
[X] PyTorch<br>
[ ] MXNet</p>
<p>Available Controllers:<br>
[X] MPI<br>
[ ] Gloo</p>
<p>Available Tensor Operations:<br>
[X] NCCL<br>
[ ] DDL<br>
[ ] CCL<br>
[X] MPI<br>
[ ] Gloo</p>

ptrblck: <p>Your code is currently a bit hard to read so could you please format it by wrapping it into three backticks ```? <img src="https://discuss.pytorch.org/images/emoji/apple/slight_smile.png?v=9" title=":slight_smile:" class="emoji" alt=":slight_smile:"></p>
<p>Is the <code>local_rank</code> always set to 0 and are thus all processes using the same device?<br>
I’m not deeply familiar with Horovod, but I assume you’ve checked <a href="https://horovod.readthedocs.io/en/stable/pytorch.html">this doc</a> already?</p>

Raj_Roy: <p>I apologize for the previous dirty code<br>
torch.cuda.device.count() gives 6 but it is using only one gpu and if I print local rank I get 0.<br>
I use this command for submitting the job</p>
<pre><code class="lang-auto">jsrun -bpacked:7 -g6 -a1 -c42 -r1 python ./pytorch_code.py
</code></pre>
<pre><code class="lang-auto">My CODE
BATCH_SIZE = 2

backend = ‘horovod’
world_size = int(os.environ[‘OMPI_COMM_WORLD_SIZE’])
world_rank = int(os.environ[‘OMPI_COMM_WORLD_RANK’])
local_rank = int(os.environ[‘OMPI_COMM_WORLD_LOCAL_RANK’])
if backend in [‘horovod’,‘ddl’]:
    import horovod.torch as hvd
    hvd.init()
torch.cuda.set_device(local_rank)
import torch.distributed as dist
if backend == ‘nccl’:
    import subprocess
    get_master = “echo $(cat {} | sort | uniq | grep -v batch | grep -v login | head -1)”.format(os.environ[‘LSB_DJOB_HOSTFILE’])
    os.environ[‘MASTER_ADDR’] = str(subprocess.check_output(get_master, shell=True))[2:-3]
    os.environ[‘MASTER_PORT’] = “23456”
    os.environ[‘WORLD_SIZE’] = os.environ[‘OMPI_COMM_WORLD_SIZE’]
    os.environ[‘RANK’] = os.environ[‘OMPI_COMM_WORLD_RANK’]
    # NCCL and MPI initialization
    print(“using ncll”)
    dist.init_process_group(backend,
    rank=world_rank, world_size=world_size)

class my_dataset(data.Dataset):
    def initialize(self, _feat_path, _tr_ros, _ss, _intra, _label_path, _file_list, _max_len):
    .
    .

    def __getitem__(self, index):
    .
    .
    .
    def __len__(self):
        return self.size

nThreads =6
train_file_list = text_file_reader(train_file)
train_dataset = my_dataset()
train_dataset.initialize( ... )
_file_list=train_file_list, _max_len=MAX_LENGTH)

train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=nThreads)

val_file_list = text_file_reader(val_file)
val_dataset = my_dataset()
val_dataset.initialize(...)
val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=nThreads)

input = torch.randn((1, FEATURES, MAX_LENGTH, MAX_LENGTH)).cuda()
import rough_copy
model = rough_copy.ResNet_custom(img_channel=FEATURES, num_classes=MAX_LENGTH * MAX_LENGTH, _depth=RESNET_DEPTH).cuda()
output = model(input)

criterion = nn.BCELoss()
optimizer = optim.Adam(model.parameters(), lr=0.0001, weight_decay=0)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=DECAY_UPDATE_INTERVAL, gamma=0.5)


if backend in [‘horovod’,‘ddl’]:
    # Horovod: wrap optimizer with DistributedOptimizer.
    optimizer = hvd.DistributedOptimizer(optimizer,
    named_parameters=model.named_parameters())
    # Horovod: broadcast parameters &amp; optimizer state.
    hvd.broadcast_parameters(model.state_dict(), root_rank=0)
    hvd.broadcast_optimizer_state(optimizer, root_rank=0)
else:
    def average_gradients(model):
    “”" Gradient averaging. “”"
    for param in model.parameters():
    dist.all_reduce(param.grad.data, op=dist.ReduceOp.SUM)
    param.grad.data /= world_size
 
for epoch in range(Current_EPOCH, EPOCH):
    model.train()
    for i, data in enumerate(train_dataloader):
    features = data[‘feat’].float().cuda()
    label = data[‘ground_truth’].float().cuda()
    optimizer.zero_grad()
    output_final = torch.squeeze(model(features))
    loss = criterion(torch.squeeze(output_val), torch.squeeze(label))
    print(loss)
    running_train_loss += loss.item()
    loss.backward()
    optimizer.step()
    scheduler.step()
    torch.save({
        'epoch': epoch,
        'model_state_dict': model.state_dict(),
        'optimizer_state_dict': optimizer.state_dict(),
        'scheduler_state_dict': scheduler.state_dict()
    }, HISTORY_FILE + "/weighths_" + str(epoch))


    Current_EPOCH = Current_EPOCH + 1

</code></pre>
<p>Output of horovodrun --check-build</p>
<pre><code class="lang-auto">
Available Frameworks:
[X] TensorFlow
[X] PyTorch
[ ] MXNet

Available Controllers:
[X] MPI
[ ] Gloo

Available Tensor Operations:
[X] NCCL
[ ] DDL
[ ] CCL
[X] MPI
[ ] Gloo
</code></pre>

ptrblck: <p>Thanks for the updated code! Is the linked example working fine for you?</p>

Raj_Roy: <p>Hi<br>
actually, the problem was with train_sampler, after using that my problem got fixed</p>