8 k80 GPU trains a classification model always stopped at epoch1, with no exception logged

username: cosinepi
cooked: <aside class="quote no-group quote-modified" data-username="cosinepi" data-post="1" data-topic="97154" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/cosinepi/48/18824_2.png" class="avatar"> cosinepi:</div>
<blockquote>
<p>import transformers<br>
from transformers import BertModel, BertTokenizer<br>
from transformers import AdamW<br>
from transformers import get_linear_schedule_with_warmup<br>
import torch<br>
import numpy as np<br>
import pandas as pd<br>
import seaborn as sns<br>
from pylab import rcParams<br>
import matplotlib.pyplot as plt<br>
from matplotlib import rc<br>
from sklearn.model_selection import train_test_split<br>
from sklearn.metrics import confusion_matrix, classification_report<br>
from collections import defaultdict<br>
from textwrap import wrap<br>
from torch import nn, optim<br>
from torch.utils.data import Dataset, DataLoader<br>
import time<br>
import logging<br>
logging.basicConfig(filename=‘bert_classifier.log’,level=logging.DEBUG)<br>
logging.debug(‘This message should go to the log file’)<br>
logging.info(‘So should this’)<br>
logging.warning(‘And this, too’)</p>
<p>import logging<br>
logging.basicConfig(level=logging.ERROR)</p>
<p>sns.set(style=‘whitegrid’, palette=‘muted’, font_scale=1.2)<br>
HAPPY_COLORS_PALETTE = [“<span class="hashtag-raw">#01BEFE</span>”, “<span class="hashtag-raw">#FFDD00</span>”, “<span class="hashtag-raw">#FF7D00</span>”, “<span class="hashtag-raw">#FF006D</span>”, “<span class="hashtag-raw">#ADFF02</span>”, “<span class="hashtag-raw">#8F00FF</span>”]<br>
sns.set_palette(sns.color_palette(HAPPY_COLORS_PALETTE))<br>
<span class="hashtag-raw">#rcParams</span>[‘figure.figsize’] = 12, 8<br>
RANDOM_SEED = 42<br>
np.random.seed(RANDOM_SEED)<br>
torch.manual_seed(RANDOM_SEED)<br>
device = torch.device(“cuda” if torch.cuda.is_available() else “cpu”)</p>
<p>df = pd.read_csv(“”)<br>
<span class="hashtag-raw">#df</span>.Text = df.Title + “.” + df.Bullet + “.” + df.Description<br>
def to_sentiment(rating):<br>
rating = int(rating)<br>
if rating &lt;= 2:<br>
return 0<br>
elif rating == 3:<br>
return 1<br>
else:<br>
return 2</p>
<p>class_names = [0,1,2]</p>
<p>PRE_TRAINED_MODEL_NAME = ‘bert-base-uncased’</p>
<p>tokenizer = BertTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)</p>
<p>MAX_LEN = 150</p>
<p>class GPReviewDataset(Dataset):</p>
<p>def <strong>init</strong>(self, reviews, targets, tokenizer, max_len):<br>
self.reviews = reviews<br>
self.targets = targets<br>
self.tokenizer = tokenizer<br>
self.max_len = max_len</p>
<p>def <strong>len</strong>(self):<br>
return len(self.reviews)</p>
<p>def <strong>getitem</strong>(self, item):<br>
review = str(self.reviews[item])<br>
target = self.targets[item]</p>
<pre><code>encoding = self.tokenizer.encode_plus(
  review,
  add_special_tokens=True,
  max_length=self.max_len,
  return_token_type_ids=False,
  pad_to_max_length=True,
  return_attention_mask=True,
  return_tensors='pt',
  truncation=True
)

return {
  'review_text': review,
  'input_ids': encoding['input_ids'].flatten(),
  'attention_mask': encoding['attention_mask'].flatten(),
  'targets': torch.tensor(target, dtype=torch.long)
}
</code></pre>
<p>df_train, df_val = train_test_split(df, test_size=0.1, random_state=RANDOM_SEED)</p>
<p>def create_data_loader(df, tokenizer, max_len, batch_size):<br>
ds = GPReviewDataset(<br>
reviews=df.Title.to_numpy(),<span class="hashtag-raw">#df</span>.Text.to_numpy(),<br>
targets=df.Flag.to_numpy(),<br>
tokenizer=tokenizer,<br>
max_len=max_len<br>
)</p>
<p>return DataLoader(<br>
ds,<br>
batch_size=batch_size,<br>
num_workers=0,<br>
drop_last=True<br>
)</p>
<p>BATCH_SIZE = 128</p>
<p>train_data_loader = create_data_loader(df_train, tokenizer, MAX_LEN, BATCH_SIZE)<br>
val_data_loader = create_data_loader(df_val, tokenizer, MAX_LEN, BATCH_SIZE)</p>
<p>bert_model = BertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)</p>
<p>class SentimentClassifier(nn.Module):</p>
<p>def <strong>init</strong>(self, n_classes):<br>
super(SentimentClassifier, self).<strong>init</strong>()<br>
self.bert = BertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)<br>
self.drop = nn.Dropout(p=0.3)<br>
self.linear = nn.Linear(self.bert.config.hidden_size, n_classes)<br>
self.out = nn.Softmax(dim=1)</p>
<p>def forward(self, input_ids, attention_mask):<br>
_, pooled_output = self.bert(<br>
input_ids=input_ids,<br>
attention_mask=attention_mask<br>
)<br>
output = self.drop(pooled_output)<br>
output_1 = self.linear(output)<br>
return self.out(output_1)</p>
<p>model = SentimentClassifier(len(class_names))<br>
model = torch.nn.DataParallel(model)<br>
model = model.to(device)</p>
<p>EPOCHS = 5</p>
<p>optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)<br>
total_steps = len(train_data_loader) * EPOCHS<br>
scheduler = get_linear_schedule_with_warmup(<br>
optimizer,<br>
num_warmup_steps=0,<br>
num_training_steps=total_steps<br>
)</p>
<p>loss_fn = nn.CrossEntropyLoss().to(device)<br>
def show_gpu(msg):<br>
“”"<br>
ref: <a href="https://discuss.pytorch.org/t/access-gpu-memory-usage-in-pytorch/3192/4" class="inline-onebox">Access GPU memory usage in Pytorch - #4 by mjstevens777</a><br>
“”"<br>
def query(field):<br>
return(subprocess.check_output(<br>
[‘nvidia-smi’, f’–query-gpu={field}‘,<br>
‘–format=csv,nounits,noheader’],<br>
encoding=‘utf-8’))<br>
def to_int(result):<br>
return int(result.strip().split(’\n’)[0])</p>
<pre><code>used = to_int(query('memory.used'))
total = to_int(query('memory.total'))
pct = used/total
logger.info('\n' + msg, f'{100*pct:2.1f}% ({used} out of {total})')   
</code></pre>
<p>def train_epoch(<br>
model,<br>
data_loader,<br>
loss_fn,<br>
optimizer,<br>
device,<br>
scheduler,<br>
n_examples<br>
):<br>
model = model.train()</p>
<p>losses = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
correct_predictions = 0<br>
index = 0<br>
for d in data_loader:<br>
try:<br>
index = index + 1<br>
input_ids = d[“input_ids”].to(device)<br>
attention_mask = d[“attention_mask”].to(device)<br>
targets = d[“targets”].to(device)</p>
<pre><code>    outputs = model(
      input_ids=input_ids,
      attention_mask=attention_mask
    )

    _, preds = torch.max(outputs, dim=1)
    loss = loss_fn(outputs, targets)

    correct_predictions += torch.sum(preds == targets)
    losses.append(loss.item())

    loss.backward()
    nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
    optimizer.step()
    scheduler.step()
    optimizer.zero_grad()
    if index%50==0:
        logging.debug("finished steps: {}".format(index))
except Exception as e:
    logging.debug("error in training step: {}".format(e))
</code></pre>
<p>return correct_predictions.double() / n_examples, np.mean(losses)</p>
<p>def eval_model(model, data_loader, loss_fn, device, n_examples):<br>
model = model.eval()</p>
<p>losses = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
correct_predictions = 0<br>
try:<br>
with torch.no_grad():<br>
for d in data_loader:<br>
input_ids = d[“input_ids”].to(device)<br>
attention_mask = d[“attention_mask”].to(device)<br>
targets = d[“targets”].to(device)</p>
<pre><code>      outputs = model(
        input_ids=input_ids,
        attention_mask=attention_mask
      )
      _, preds = torch.max(outputs, dim=1)
      
      #print(outputs)
      loss = loss_fn(outputs, targets)

      correct_predictions += torch.sum(preds == targets)
      losses.append(loss.item())
</code></pre>
<p>except Exception as e:<br>
logger.info(“eval error {}”.format(e))</p>
<p>return correct_predictions.double() / n_examples, np.mean(losses)</p>
<p>history = defaultdict(list)<br>
best_accuracy = 0<br>
start_time = time.time()<br>
show_gpu(‘Initial GPU memory usage:’)<br>
for epoch in range(EPOCHS):</p>
<p>logging.info(f’Epoch {epoch + 1}/{EPOCHS}‘)<br>
logging.info(’-’ * 10)</p>
<p>train_acc, train_loss = train_epoch(<br>
model,<br>
train_data_loader,<br>
loss_fn,<br>
optimizer,<br>
device,<br>
scheduler,<br>
len(df_train)<br>
)<br>
show_gpu(‘GPU memory usage after training model in epoch {}’.format(epoch))</p>
<p>logging.info(f’Train loss {train_loss} accuracy {train_acc}')<br>
logging.info(“time taken from start: {}”.format(time.time() - start_time))</p>
<p>val_acc, val_loss = eval_model(<br>
model,<br>
val_data_loader,<br>
loss_fn,<br>
device,<br>
len(df_val)<br>
)</p>
<p>print(f’Val   loss {val_loss} accuracy {val_acc}')<br>
logging.info(“val accuracy {}, validation_loss {}”.format(val_acc, val_loss))</p>
<p>history[‘train_acc’].append(train_acc)<br>
history[‘train_loss’].append(train_loss)<br>
history[‘val_acc’].append(val_acc)<br>
history[‘val_loss’].append(val_loss)</p>
<p>if val_acc &gt; best_accuracy:<br>
torch.save(model.state_dict(), ‘best_model_state_{}.bin’.format(epoch))<br>
best_accuracy = val_acc</p>
</blockquote>
</aside>
<p>Hey guys tried to fine tune a bert classification model with 1.06 million data, but the training always stuck after it finishes the first epoch with no exception logged. can anybody see what is the problem with this? or any debugging idea? thanks a lot!</p>

username: ptrblck
cooked: <p>Could you remove the dataloading and check if your training routine would get stuck after training on random data?<br>
If that’s not the case, could you iterate over the <code>DataLoader</code> alone without the model training and see, if this might cause the issue?</p>