Issue with pytorch tensors and multiple GPUs when using DataParallel

Jc112358: <p>I have a large ML code that I’ve been writing for a few months and I’ve started the process of try to parallelize the data side of things to work with multiple GPUs. To start, the code works perfectly when using a single GPU; the issue comes when using multiple GPUs.</p>
<p>The error is as follows: <code>RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:1 and cuda:0! (when checking argument for argument index in method wrapper_CUDA__index_select)</code></p>
<p>Snippets of relevant code can be found below:</p>
<p><strong>Model file</strong></p>
<pre><code class="lang-auto">import torch
from torch import nn
from functools import partial
import copy

from ..mlp import MLP
from ..basis import gaussian, bessel
from ..conv import GatedGCN


class Encoder(nn.Module):
    """ALIGNN/ALIGNN-d Encoder.
    The encoder must take a PyG graph object `data` and output the same `data`
    with additional fields `h_atm`, `h_bnd`, and `h_ang` that correspond to the atom, bond, and angle embedding.

    The input `data` must have three fields `x_atm`, `x_bnd`, and `x_ang` that describe the atom type
    (in onehot vectors), the bond lengths, and bond/dihedral angles (in radians).
    """

    def __init__(self, num_species, cutoff, dim=128, dihedral=False):
        super().__init__()
        self.num_species = num_species
        self.cutoff = cutoff
        self.dim = dim
        self.dihedral = dihedral

        self.embed_atm = nn.Sequential(MLP([num_species, dim, dim], act=nn.SiLU()), nn.LayerNorm(dim))
        self.embed_bnd = partial(bessel, start=0, end=cutoff, num_basis=dim)
        self.embed_ang = self.embed_ang_with_dihedral if dihedral else self.embed_ang_without_dihedral

    def embed_ang_with_dihedral(self, x_ang, mask_dih_ang):
        cos_ang = torch.cos(x_ang)
        sin_ang = torch.sin(x_ang)

        h_ang = torch.zeros([len(x_ang), self.dim], device=x_ang.device)
        h_ang[~mask_dih_ang, :self.dim // 2] = gaussian(cos_ang[~mask_dih_ang], start=-1, end=1,
                                                        num_basis=self.dim // 2)

        h_cos_ang = gaussian(cos_ang[mask_dih_ang], start=-1, end=1, num_basis=self.dim // 4)
        h_sin_ang = gaussian(sin_ang[mask_dih_ang], start=-1, end=1, num_basis=self.dim // 4)
        h_ang[mask_dih_ang, self.dim // 2:] = torch.cat([h_cos_ang, h_sin_ang], dim=-1)

        return h_ang

    def embed_ang_without_dihedral(self, x_ang, mask_dih_ang):
        cos_ang = torch.cos(x_ang)
        return gaussian(cos_ang, start=-1, end=1, num_basis=self.dim)

    def forward(self, data):
        # Embed atoms
        data.h_atm = self.embed_atm(data.x_atm)

        # Embed bonds
        data.h_bnd = self.embed_bnd(data.x_bnd)

        # Embed angles
        data.h_ang = self.embed_ang(data.x_ang, data.mask_dih_ang)

        return data


class Processor(nn.Module):
    """ALIGNN Processor.
    The processor updates atom, bond, and angle embeddings.
    """

    def __init__(self, num_convs, dim):
        super().__init__()
        self.num_convs = num_convs
        self.dim = dim

        self.atm_bnd_convs = nn.ModuleList([copy.deepcopy(GatedGCN(dim, dim)) for _ in range(num_convs)])
        self.bnd_ang_convs = nn.ModuleList([copy.deepcopy(GatedGCN(dim, dim)) for _ in range(num_convs)])

    def forward(self, data):
        edge_index_G = data.edge_index_G
        edge_index_A = data.edge_index_A

        for i in range(self.num_convs):
            data.h_bnd, data.h_ang = self.bnd_ang_convs[i](data.h_bnd, edge_index_A, data.h_ang)
            data.h_atm, data.h_bnd = self.atm_bnd_convs[i](data.h_atm, edge_index_G, data.h_bnd)

        return data


class Decoder(nn.Module):
    def __init__(self, node_dim, out_dim):
        super().__init__()
        self.node_dim = node_dim
        self.out_dim = out_dim
        self.decoder = MLP([node_dim, node_dim, out_dim], act=nn.SiLU())

    def forward(self, data):
        return self.decoder(data.h_atm)


class ALIGNN(nn.Module):
    """ALIGNN model.
    Can optinally encode dihedral angles.
    """

    def __init__(self, encoder, processor, decoder):
        super().__init__()
        self.encoder = encoder
        self.processor = processor
        self.decoder = decoder

    def forward(self, data):
        data = self.encoder(data)
        data = self.processor(data)
        return self.decoder(data)
</code></pre>
<p><strong>Training file</strong></p>
<pre><code class="lang-auto">from tqdm.notebook import trange
from datetime import datetime
import glob
import sys
import os

def train(loader,model,parameters,PIN_MEMORY=False):
    model.train()
    total_loss = 0.0
    model = nn.DataParallel(model, device_ids=[0, 1]).cuda()
    #model = model.to(parameters['device'])
    optimizer = torch.optim.AdamW(model.module.processor.parameters(), lr=parameters['LEARN_RATE'])

    #model = model.to(parameters['device'])
    loss_fn = torch.nn.MSELoss()
    for i,data in enumerate(loader, 0):
        optimizer.zero_grad(set_to_none=True)
        #data = data.to(parameters['device'], non_blocking=PIN_MEMORY)
        data = data.cuda()
        #encoding = model.encoder(data)
        #proc = model.processor(encoding.module)
        #atom_contrib, bond_contrib, angle_contrib = model.decoder(proc.module)
        atom_contrib, bond_contrib, angle_contrib = model(data)

        all_sum = atom_contrib.sum() + bond_contrib.sum() + angle_contrib.sum()

        loss = loss_fn(all_sum, data.y[0][0])
        loss.backward()
        optimizer.step()
        total_loss += loss.item()
    return total_loss / len(loader)

def run_training(data,parameters,model):
    follow_batch = ['x_atm', 'x_bnd', 'x_ang'] if hasattr(data['training'][0], 'x_ang') else ['x_atm']
    loader_train = DataLoader(data['training'], batch_size=parameters['BATCH_SIZE'], shuffle=True, follow_batch=follow_batch)
    loader_valid = DataLoader(data['validation'], batch_size=parameters['BATCH_SIZE'], shuffle=False)

    L_train, L_valid = [], []
    min_loss_train = 1.0E30
    min_loss_valid = 1.0E30

    stats_file = open(os.path.join(parameters['model_dir'],'loss.data'),'w')
    stats_file.write('Training_loss     Validation loss\n')
    stats_file.close()
    for ep in range(parameters['num_epochs']):
        stats_file = open(os.path.join(parameters['model_dir'], 'loss.data'), 'a')
        print('Epoch ',ep,' of ',parameters['num_epochs'])
        sys.stdout.flush()
        loss_train = train(loader_train, model, parameters);
        L_train.append(loss_train)
        loss_valid = test_non_intepretable(loader_valid, model, parameters)
        L_valid.append(loss_valid)
        stats_file.write(str(loss_train) + '     ' + str(loss_valid) + '\n')
        if loss_train &lt; min_loss_train:
            min_loss_train = loss_train
            if loss_valid &lt; min_loss_valid:
                min_loss_valid = loss_valid
                if parameters['remove_old_model']:
                    model_name = glob.glob(os.path.join(parameters['model_dir'], 'model_*'))
                    if len(model_name) &gt; 0:
                        os.remove(model_name[0])
                now = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
                print('Min train loss: ', min_loss_train, ' min valid loss: ', min_loss_valid, ' time: ', now)
                torch.save(model.state_dict(), os.path.join(parameters['model_dir'], 'model_' + str(now)))
        stats_file.close()
        if loss_train &lt; parameters['train_tolerance'] and loss_valid &lt; parameters['train_tolerance']:
            print('Validation and training losses satisy set tolerance...exiting training loop...')
            break
</code></pre>
<p>There are many other files but I think these are the relevant ones for this specific problem, but I’m happy to include for code if needed. My data is stored as a graph and batched by a DataLoader and looks like this (when batched two at a time):</p>
<p><code>Graph_DataBatch(atoms=[2], edge_index_G=[2, 89966], edge_index_A=[2, 1479258], x_atm=[5184, 5], x_atm_batch=[5184], x_atm_ptr=[3], x_bnd=[89966], x_bnd_batch=[89966], x_bnd_ptr=[3], x_ang=[1479258], x_ang_batch=[1479258], x_ang_ptr=[3], mask_dih_ang=[1479258], atm_amounts=[6], bnd_amounts=[6], ang_amounts=[6], y=[179932, 1])</code></p>
<p>The problematic line is in the Encoder during its forward function:</p>
<pre><code class="lang-auto">def forward(self, data):
        # Embed atoms
        data.h_atm = self.embed_atm(data.x_atm)

        # Embed bonds
        data.h_bnd = self.embed_bnd(data.x_bnd)

        # Embed angles
        data.h_ang = self.embed_ang(data.x_ang, data.mask_dih_ang)

        return data
</code></pre>
<p>Specifically, <code>data.h_atm = self.embed_atm(data.x_atm)</code>. To briefly explain what I’m trying to do, I am loading in a bunch of graphs into the Dataloader, which are batched and then fed into the model for training:</p>
<pre><code class="lang-auto">def train(loader,model,parameters,PIN_MEMORY=False):
    model.train()
    total_loss = 0.0
    model = nn.DataParallel(model, device_ids=[0, 1]).cuda()
    #model = model.to(parameters['device'])
    optimizer = torch.optim.AdamW(model.module.processor.parameters(), lr=parameters['LEARN_RATE'])

    #model = model.to(parameters['device'])
    loss_fn = torch.nn.MSELoss()
    for i,data in enumerate(loader, 0):
        optimizer.zero_grad(set_to_none=True)
        #data = data.to(parameters['device'], non_blocking=PIN_MEMORY)
        data = data.cuda()
        #encoding = model.encoder(data)
        #proc = model.processor(encoding.module)
        #atom_contrib, bond_contrib, angle_contrib = model.decoder(proc.module)
        atom_contrib, bond_contrib, angle_contrib = model(data)
</code></pre>
<p>My understanding is that I have sent my batched data to the GPU and my model parameters are on the GPU and DataParallel should take care of splitting my data up and sending everything to each GPU automatically.</p>
<p>My question can be broken into a few parts: (1) Is this understanding correct? (2) Does my code actually seem like its doing this, and (3) does this error have anything to do with that, and if not, what is this error trying to tell me? I don’t expect anyone to fix my code for me, but I would like to understand why this error is happening, because I think I’m misunderstanding the underlying logic of how DataParallel is taking my data and sending it to the GPU. I’m happy to provide any details you might need to better understand this problem.</p>
<p>I have tried to better understand the line that breaks: <code>data.h_atm = self.embed_atm(data.x_atm)</code> by printing out where <code>data.x_atm</code> actually is when inside of the forward function, which should be after DataParallel has partitioned the data and I get this for all tensors:</p>
<p><code>tensor([[1., 0., 0., 0., 0.],</code><br>
<code>[1., 0., 0., 0., 0.],</code><br>
<code>[1., 0., 0., 0., 0.],</code><br>
<code>...,</code><br>
<code>[0., 0., 1., 0., 0.],</code><br>
<code>[0., 0., 1., 0., 0.],</code><br>
<code>[0., 0., 1., 0., 0.]], device='cuda:0')</code></p>
<p>I think this telling me that all of my data is on GPU 0 despite the model being on both GPU 0 and 1 (when running on 2 GPUs), which I’ve confirmed by using <code>nvidia-smi</code> and observing both GPUs at about half memory consumption. I have also tried various combinations of calling <code>X.to_device('cuda')</code> or <code>X.cuda()</code>, where X is my graph data, but nothing seems to make any difference to the tensor print out.</p>

ptrblck: <p>These issues are often caused by explicit data movement added to the <code>forward</code> methods of your modules. I don’t see anything obviously wrong, but check if you are moving any tensor to e.g. the default device.</p>
<p>If you cannot narrow down which input is on the wrong device, add debug <code>print</code> statements showing the device of your input data as well as the parameters.</p>
<aside class="quote no-group" data-username="Jc112358" data-post="1" data-topic="206088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/j/e68b1a/48.png" class="avatar"> Jc112358:</div>
<blockquote>
<p>My understanding is that I have sent my batched data to the GPU and my model parameters are on the GPU and DataParallel should take care of splitting my data up and sending everything to each GPU automatically.</p>
</blockquote>
</aside>
<p>Yes, this is correct. <code>DataParallel</code> will clone the model onto each device in the forward pass and will split the inputs sending the chunks to the corresponding device. This adds a lot of overhead, which is why we generally recommend using <code>DistributedDataParallel</code>.</p>
<aside class="quote no-group" data-username="Jc112358" data-post="1" data-topic="206088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/j/e68b1a/48.png" class="avatar"> Jc112358:</div>
<blockquote>
<p>I think this telling me that all of my data is on GPU 0 despite the model being on both GPU 0 and 1 (when running on 2 GPUs),</p>
</blockquote>
</aside>
<p>Yes, this could be the case. I don’t know what <code>data</code> is, but in case it’s not a plain tensor, I guess their internal data won’t be moved to the right device. You could try to move it explicitly by using any parameters, e.g.:</p>
<pre data-code-wrap="python"><code class="lang-python">data.h_atm = data.h_atm.t(next(self.parameters()).device)
</code></pre>

Jc112358: <p>Thanks for your input. You were right that the tensors in data were not being moved to the correct device. The solution was the manually call <code>data = data.cuda()</code> inside of every forward call. What I don’t understand is that inside of the training loop when iterating through the batched data within DataLoader, I call <code>data = data.cuda()</code>, but when that data is passed through a model’s forward function that data isn’t sent to the correct GPU. Is the issue that, when the call <code>data = data.cuda()</code> is made inside of the DataLoader loop, it isn’t in communication with DataParallel at that point and therefore is just sent to the default GPU device?</p>

ptrblck: <aside class="quote no-group" data-username="Jc112358" data-post="3" data-topic="206088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/j/e68b1a/48.png" class="avatar"> Jc112358:</div>
<blockquote>
<p>The solution was the manually call <code>data = data.cuda()</code> inside of every forward call.</p>
</blockquote>
</aside>
<p>Calling <code>data.cuda()</code> inside the <code>forward</code> method of a module wrapped in <code>nn.DataParallel</code> will move it to the currently set device, which is where the parameters were also moved to.<br>
<code>nn.DataParallel</code> will use <code>parallel_apply</code> <a href="https://github.com/pytorch/pytorch/blob/1cae60a87e5bdda8bcf55724a862eeed98a9747e/torch/nn/parallel/data_parallel.py#L194">here</a> which will set the device <a href="https://github.com/pytorch/pytorch/blob/1cae60a87e5bdda8bcf55724a862eeed98a9747e/torch/nn/parallel/parallel_apply.py#L92">here</a>.<br>
If you do not specify a device ID, the set device will be used, and will thus fix your error.</p>
<p>Calling <code>data.cuda()</code> outside will also use the currently set device, which is most likely the default <code>cuda:0</code> device, and will thus fail.</p>

Jc112358: <p>Got it. So, I just got done converting my code to DistributedDataParallel and it is much faster and everything seems to be working. What I don’t understand is why the <code>data = data.cuda()</code> calls aren’t needed in the forward functions when using DDP but are needed when using DP?</p>

ptrblck: <p><code>DDP</code> will use a single process per device and assumes each process uses its corresponding GPU only. Usually you would call <code>torch.cuda.set_device</code> as the beginning of your script to make sure all <code>.cuda()</code> or <code>to("cuda")</code> calls are using this specified device.</p>
<p>Great to hear <code>DDP</code> is giving you a speedup!</p>

Jc112358: <p>So, I wanted to follow up on this and get some additional feedback. I have the code working on our local GPU machine and it runs smoothly. I’ve since moved the code to our HPC cluster and I’m running into a problem regarding Exclusive vs Default mode on the GPU. Essentially, the error is:</p>
<pre><code class="lang-auto">RuntimeError: CUDA error: CUDA-capable device(s) is/are busy or unavailable
</code></pre>
<p>After speaking with our HPC support staff the issue is that when in Exclusive mode each GPU cannot have multiple processes spawned on it. However, when running the below line in my main script:</p>
<pre><code class="lang-auto">mp.spawn(run_training, args=(partitioned_data, ml.parameters, ml.model,ml), nprocs=ml_parameters['world_size'],
                 join=True)
</code></pre>
<p>And setting world_size equal to the number of GPUs, my understanding is that each GPU should only have 1 processes running on it. Is that correct? The rest of my code that uses DDP is below:</p>
<pre><code class="lang-auto">def ddp_setup(rank: int,world_size):
    """
    Args:
    rank: Unique identifier of each process
    world_size: Total number of processes
    """
    os.environ["MASTER_ADDR"] = "localhost"
    os.environ["MASTER_PORT"] = "12355"
    torch.cuda.set_device(rank)
    init_process_group(backend="gloo", rank=rank, world_size=world_size)
</code></pre>
<p>and</p>
<pre><code class="lang-auto">loader_train = DataLoader(data['training'], batch_size=parameters['BATCH_SIZE'], shuffle=False, follow_batch=follow_batch,sampler=DistributedSampler(data['training']))
</code></pre>

ptrblck: <aside class="quote no-group" data-username="Jc112358" data-post="7" data-topic="206088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/j/e68b1a/48.png" class="avatar"> Jc112358:</div>
<blockquote>
<p>After speaking with our HPC support staff the issue is that when in Exclusive mode each GPU cannot have multiple processes spawned on it.</p>
</blockquote>
</aside>
<p>I assume you are referring to <code>EXCLUSIVE_PROCESS</code> set via <code>nvidia-smi</code>. If so, then only a single process is allowed to initialize a CUDA context on this device and multiple threads may submit work to this context.<br>
If your approach tries to create multiple CUDA contexts on the same device, your script is wrong and you might not be setting the device correctly as mentioned above:</p>
<aside class="quote no-group quote-modified" data-username="ptrblck" data-post="6" data-topic="206088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/ptrblck/48/1823_2.png" class="avatar"> ptrblck:</div>
<blockquote>
<p>[…] and assumes each process uses its corresponding GPU only. Usually you would call <code>torch.cuda.set_device</code> as the beginning of your scrip […]</p>
</blockquote>
</aside>

Jc112358: <p><code>Usually you would call </code>torch.cuda.set_device<code> as the beginning of your scrip […]</code></p>
<p>Isn’t that what I’m doing here though:</p>
<pre><code class="lang-auto">def ddp_setup(rank: int,world_size):
    """
    Args:
    rank: Unique identifier of each process
    world_size: Total number of processes
    """
    os.environ["MASTER_ADDR"] = "localhost"
    os.environ["MASTER_PORT"] = "12355"
    torch.cuda.set_device(rank)
    init_process_group(backend="gloo", rank=rank, world_size=world_size)
</code></pre>
<p>I’m also confused about this:<br>
<code>If so, then only a single process is allowed to initialize a CUDA context on this device and multiple threads may submit work to this context.</code></p>
<p>If I only have 2 GPUs and request 2 processes which should be set to either rank 0 or rank 1, then how can either GPU say that there is more than 1 process running on it? Is there a way that I can debug this somehow via simple print commands, because I don’t think I understand what is happening at a fundamental level? If it helps, here’s the full error:</p>
<pre><code class="lang-auto">Traceback (most recent call last):
  File "&lt;string&gt;", line 1, in &lt;module&gt;
  File "/share/pkg.7/python3/3.9.4/install/lib/python3.9/multiprocessing/spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "/share/pkg.7/python3/3.9.4/install/lib/python3.9/multiprocessing/spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)                                                                             File "/project/cmdlab/software/python3.9.4/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 147, in rebuild_cuda_tensor                                                                                                     storage = storage_cls._new_shared_cuda(
  File "/project/cmdlab/software/python3.9.4/lib/python3.9/site-packages/torch/storage.py", line 1085, in _new_shared_cuda
    return torch.UntypedStorage._new_shared_cuda(*args, **kwargs)
RuntimeError: CUDA error: CUDA-capable device(s) is/are busy or unavailable
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
</code></pre>