How to run on multi GPU

Gorgen: <p>problem about how multi GPU. I run the command “CUDA_VISIBLE_DEVICES=0,1 python train.py” and specify “os.environ[“CUDA_VISIBLE_DEVICES”]= ‘2’” and "    model = model.cuda(2) ". is it true? that can work on multi GPU? thanks, best  wishes</p>
<p>run the command as follows " CUDA_VISIBLE_DEVICES=0,1 python train.py"</p>
<p>the train.py is as followed:" # -<em>- coding: utf-8 -</em>-<br>
import os<br>
os.environ[“CUDA_VISIBLE_DEVICES”]= ‘2’<br>
from tqdm import tqdm<br>
import os<br>
import random<br>
import torch<br>
import torch.nn as nn</p>
<p>from transformers import RobertaTokenizer<br>
from ERC_dataset import MELD_loader, Emory_loader, IEMOCAP_loader, DD_loader<br>
from model import ERC_model</p>
<h1>
<a name="from-erccombined-import-erc_model-1" class="anchor" href="#from-erccombined-import-erc_model-1"></a>from ERCcombined import ERC_model</h1>
<p>from torch.utils.data import Dataset, DataLoader<br>
from transformers import get_linear_schedule_with_warmup<br>
import pdb<br>
import argparse, logging<br>
from sklearn.metrics import precision_recall_fscore_support</p>
<p>from utils import make_batch_roberta, make_batch_bert, make_batch_gpt</p>
<p>def CELoss(pred_outs, labels):<br>
“”"<br>
pred_outs: [batch, clsNum]<br>
labels: [batch]<br>
“”"<br>
loss = nn.CrossEntropyLoss()<br>
loss_val = loss(pred_outs, labels)<br>
return loss_val</p>
<h2>
<a name="finetune-robeta-large-2" class="anchor" href="#finetune-robeta-large-2"></a>finetune RoBETa-large</h2>
<p>def main():<br>
“”“Dataset Loading”“”<br>
batch_size = args.batch<br>
dataset = args.dataset<br>
dataclass = args.cls<br>
sample = args.sample<br>
model_type = args.pretrained<br>
freeze = args.freeze<br>
initial = args.initial</p>
<pre><code>dataType = 'multi'
if dataset == 'MELD':
    if args.dyadic:
        dataType = 'dyadic'
    else:
        dataType = 'multi'
    data_path = './dataset/MELD/'+dataType+'/'
    DATA_loader = MELD_loader
elif dataset == 'EMORY':
    data_path = './dataset/EMORY/'
    DATA_loader = Emory_loader
elif dataset == 'iemocap':
    data_path = './dataset/iemocap/'
    DATA_loader = IEMOCAP_loader
elif dataset == 'dailydialog':
    data_path = './dataset/dailydialog/'
    DATA_loader = DD_loader    
    
if 'roberta' in model_type:
    make_batch = make_batch_roberta
elif model_type == 'bert-large-uncased':
    make_batch = make_batch_bert
else:
    make_batch = make_batch_gpt  
    
if freeze:
    freeze_type = 'freeze'
else:
    freeze_type = 'no_freeze'
    
train_path = data_path + dataset+'_train.txt'
dev_path = data_path + dataset+'_dev.txt'
test_path = data_path + dataset+'_test.txt'
        
train_dataset = DATA_loader(train_path, dataclass)
if sample &lt; 1.0:
    train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=make_batch)
else:
    train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4, collate_fn=make_batch)
train_sample_num = int(len(train_dataloader)*sample)

dev_dataset = DATA_loader(dev_path, dataclass)
dev_dataloader = DataLoader(dev_dataset, batch_size=1, shuffle=False, num_workers=4, collate_fn=make_batch)

test_dataset = DATA_loader(test_path, dataclass)
test_dataloader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=4, collate_fn=make_batch)

"""logging and path"""
save_path = os.path.join(dataset+'_models', model_type, initial, freeze_type, dataclass, str(sample))

print("###Save Path### ", save_path)
log_path = os.path.join(save_path, 'train.log')
if not os.path.exists(save_path):
    os.makedirs(save_path)
fileHandler = logging.FileHandler(log_path)

logger.addHandler(streamHandler)
logger.addHandler(fileHandler)    
logger.setLevel(level=logging.DEBUG)      

"""Model Loading"""
if 'gpt2' in model_type:
    last = True
else:
    last = False
    
print('DataClass: ', dataclass, '!!!') # emotion
clsNum = len(train_dataset.labelList)
model = ERC_model(model_type, clsNum, last, freeze, initial)
model = model.cuda(2)    
model. Train() 

"""Training Setting"""        
training_epochs = args.epoch
save_term = int(training_epochs/5)
max_grad_norm = args.norm
lr = args.lr
num_training_steps = len(train_dataset)*training_epochs
num_warmup_steps = len(train_dataset)
# optimizer = torch.optim.AdamW(model.parameters(), lr=lr) # , eps=1e-06, weight_decay=0.01
optimizer = torch.optim.AdamW(model.train_params, lr=lr) # , eps=1e-06, weight_decay=0.01
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps)

"""Input &amp; Label Setting"""
best_dev_fscore, best_test_fscore = 0, 0
best_dev_fscore_macro, best_dev_fscore_micro, best_test_fscore_macro, best_test_fscore_micro = 0, 0, 0, 0    
best_epoch = 0
for epoch in tqdm(range(training_epochs)):
    model.train() 
    for i_batch, data in enumerate(train_dataloader):
        if i_batch &gt; train_sample_num:
            print(i_batch, train_sample_num)
            break
        
        """Prediction"""
        batch_input_tokens, batch_labels, batch_speaker_tokens = data
        batch_input_tokens, batch_labels = batch_input_tokens.cuda(), batch_labels.cuda()
        
        pred_logits = model(batch_input_tokens, batch_speaker_tokens)

        """Loss calculation &amp; training"""
        loss_val = CELoss(pred_logits, batch_labels)
        
        loss_val.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)  # Gradient clipping is not in AdamW anymore (so you can use amp without issue)
        optimizer.step()
        scheduler.step()
        optimizer.zero_grad()
        
    """Dev &amp; Test evaluation"""
    model.eval()
    if dataset == 'dailydialog': # micro &amp; macro
        dev_acc, dev_pred_list, dev_label_list = _CalACC(model, dev_dataloader)
        dev_pre_macro, dev_rec_macro, dev_fbeta_macro, _ = precision_recall_fscore_support(dev_label_list, dev_pred_list, average='macro')
        dev_pre_micro, dev_rec_micro, dev_fbeta_micro, _ = precision_recall_fscore_support(dev_label_list, dev_pred_list, labels=[0,1,2,3,5,6], average='micro') # neutral x
        
        dev_fscore = dev_fbeta_macro+dev_fbeta_micro

        """Best Score &amp; Model Save"""
        if dev_fscore &gt; best_dev_fscore_macro + best_dev_fscore_micro:
            best_dev_fscore_macro = dev_fbeta_macro                
            best_dev_fscore_micro = dev_fbeta_micro
            
            test_acc, test_pred_list, test_label_list = _CalACC(model, test_dataloader)
            test_pre_macro, test_rec_macro, test_fbeta_macro, _ = precision_recall_fscore_support(test_label_list, test_pred_list, average='macro')
            test_pre_micro, test_rec_micro, test_fbeta_micro, _ = precision_recall_fscore_support(test_label_list, test_pred_list, labels=[0,1,2,3,5,6], average='micro') # neutral x                
            
            best_epoch = epoch
            _SaveModel(model, save_path)
    else: # weight
        dev_acc, dev_pred_list, dev_label_list = _CalACC(model, dev_dataloader)
        dev_pre, dev_rec, dev_fbeta, _ = precision_recall_fscore_support(dev_label_list, dev_pred_list, average='weighted')

        """Best Score &amp; Model Save"""
        if dev_fbeta &gt; best_dev_fscore:
            best_dev_fscore = dev_fbeta
            
            test_acc, test_pred_list, test_label_list = _CalACC(model, test_dataloader)
            test_pre, test_rec, test_fbeta, _ = precision_recall_fscore_support(test_label_list, test_pred_list, average='weighted')                
            
            best_epoch = epoch
            _SaveModel(model, save_path)
    
    if epoch % 5 == 0:
        logger.info('Epoch: {}'.format(epoch))
        if dataset == 'dailydialog': # micro &amp; macro
            logger.info('Devleopment ## accuracy: {}, macro-fscore: {}, micro-fscore: {}'.format(dev_acc, dev_fbeta_macro, dev_fbeta_micro))
            logger.info('') 
        else:
            logger.info('Devleopment ## accuracy: {}, precision: {}, recall: {}, fscore: {}'.format(dev_acc, dev_pre, dev_rec, dev_fbeta))
            logger.info('')
    
if dataset == 'dailydialog': # micro &amp; macro
    logger.info('Final Fscore ## test-accuracy: {}, test-macro: {}, test-micro: {}, test_epoch: {}'.format(test_acc, test_fbeta_macro, test_fbeta_micro, best_epoch))
else:
    logger.info('Final Fscore ## test-accuracy: {}, test-fscore: {}, test_epoch: {}'.format(test_acc, test_fbeta, best_epoch))         
</code></pre>
<p>def _CalACC(model, dataloader):<br>
model.eval()<br>
correct = 0<br>
label_list = []<br>
pred_list = []</p>
<pre><code># label arragne
with torch.no_grad():
    for i_batch, data in enumerate(dataloader):            
        """Prediction"""
        batch_input_tokens, batch_labels, batch_speaker_tokens = data
        batch_input_tokens, batch_labels = batch_input_tokens.cuda(), batch_labels.cuda()
        
        pred_logits = model(batch_input_tokens, batch_speaker_tokens) # (1, clsNum)
        
        """Calculation"""    
        pred_label = pred_logits.argmax(1).item()
        true_label = batch_labels.item()
        
        pred_list.append(pred_label)
        label_list.append(true_label)
        if pred_label == true_label:
            correct += 1
    acc = correct/len(dataloader)
return acc, pred_list, label_list
</code></pre>
<p>def _SaveModel(model, path):<br>
if not os.path.exists(path):<br>
os.makedirs(path)<br>
torch.save(model.state_dict(), os.path.join(path, ‘model.bin’))</p>
<p>if <strong>name</strong> == ‘<strong>main</strong>’:<br>
torch.cuda.empty_cache()</p>
<pre><code>"""Parameters"""
parser  = argparse.ArgumentParser(description = "Emotion Classifier" )
parser.add_argument( "--batch", type=int, help = "batch_size", default = 1)

parser.add_argument( "--epoch", type=int, help = 'training epohcs', default = 10) # 12 for iemocap
parser.add_argument( "--norm", type=int, help = "max_grad_norm", default = 10)
parser.add_argument( "--lr", type=float, help = "learning rate", default = 1e-6) # 1e-5
parser.add_argument( "--sample", type=float, help = "sampling trainign dataset", default = 1.0) # 

parser.add_argument( "--dataset", help = 'MELD or EMORY or iemocap or dailydialog', default = 'MELD')

parser.add_argument( "--pretrained", help = 'roberta-large or bert-large-uncased or gpt2 or gpt2-large or gpt2-medium', default = 'roberta-large')    
parser.add_argument( "--initial", help = 'pretrained or scratch', default = 'pretrained')
parser.add_argument('-dya', '--dyadic', action='store_true', help='dyadic conversation')
parser.add_argument('-fr', '--freeze', action='store_true', help='freezing PM')
parser.add_argument( "--cls", help = 'emotion or sentiment', default = 'emotion')
    
args = parser.parse_args()

logger = logging.getLogger(__name__)
streamHandler = logging.StreamHandler()

main()
"
</code></pre>

ptrblck: <aside class="quote no-group" data-username="Gorgen" data-post="1" data-topic="175297">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/g/f14d63/48.png" class="avatar"> Gorgen:</div>
<blockquote>
<p>I run the command “CUDA_VISIBLE_DEVICES=0,1 python train.py” and specify “os.environ[“CUDA_VISIBLE_DEVICES”]= ‘2’” and " model = model.cuda(2) ". is it true? that can work on multi GPU? thanks, best wishes</p>
</blockquote>
</aside>
<p>No, this should not work for a few reasons:</p>
<ul>
<li>Calling <code>os.environ[“CUDA_VISIBLE_DEVICES”]= ‘2’</code> inside the script will override the previously specified environment variable, so what’s the reason to set it in the terminal?</li>
<li><code>model.cuda(2)</code> will fail, since <code>CUDA_VISIBLE_DEVICES</code> maps the GPUs to ids starting with <code>0</code>. E.g. <code>CUDA_VISIBLE_DEVICES=2 python script.py</code> will allow <code>script.py</code> to use GPU2 mapped to <code>cuda:0</code> inside the script.</li>
</ul>

Gorgen: <p>Many thanks for your detailed advice. I will try to solve it.</p>