Kaggle competition: Toxic Comment Classification Challenge
cols = ['Id', 'parentesco1', 'v2a1', 'v18q', 'hacapo', 'rooms', 'r4t3', 'hhsize', 'escolari', 'epared2',
'epared3', 'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5', 'Target']<filter>
!pip install transformers
if not sys.warnoptions:
warnings.simplefilter("ignore" )
v2a1_null_heads.query('tipovivi1 != 1 & tipovivi2 != 1 & tipovivi3 != 1 & tipovivi4 != 1 & tipovivi5 != 1')
<feature_engineering>
sample_submission = pd.read_csv('/kaggle/input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 test_labels = pd.read_csv('//kaggle/input/jigsaw-toxic-comment-classification-challenge/test_labels.csv')
 train = pd.read_csv('//kaggle/input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('//kaggle/input/jigsaw-toxic-comment-classification-challenge/test.csv' )
train.loc[train['v2a1'].isnull() & train['tipovivi1'] == 1, 'v2a1'] = 0<feature_engineering>
Y = train[['toxic','severe_toxic','obscene','threat','insult','identity_hate']].astype(float)
 X = train['comment_text']
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.33, random_state=42 )
test.loc[test['v2a1'].isnull() & test['tipovivi1'] == 1, 'v2a1'] = 0<feature_engineering>
X_train = X_train.values
X_test = X_test.values
y_train = y_train.values
y_test = y_test.values
train.loc[train['v2a1'].isnull() &(train['Target']== 1), 'v2a1'] = a
train.loc[train['v2a1'].isnull() &(train['Target']== 2), 'v2a1'] = b
train.loc[train['v2a1'].isnull() &(train['Target']== 3), 'v2a1'] = c
train.loc[train['v2a1'].isnull() &(train['Target']== 4), 'v2a1'] = d
train.loc[train['v2a1'].isnull() ]<feature_engineering>
def accuracy_thresh(y_pred, y_true, thresh:float=0.4, sigmoid:bool=True):
"Compute accuracy when `y_pred` and `y_true` are the same size."
if sigmoid: y_pred = y_pred.sigmoid()
return np.mean(((y_pred>thresh ).float() ==y_true.float() ).float().cpu().numpy() , axis=1 ).sum()
test.loc[test['v2a1'].isnull() , 'v2a1'] =(a+b+c+d)/4
test.loc[test['v2a1'].isnull() ]<count_unique_values>
config = DistilBertConfig(vocab_size_or_config_json_file=32000, hidden_size=768,dropout=0.1,num_labels=6,
num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072 )
h_ids = v18q1_null['idhogar'].unique() ; h_ids.shape<feature_engineering>
class DistilBertForSequenceClassification(nn.Module):
def __init__(self, config):
super().__init__()
self.num_labels = config.num_labels
self.distilbert = DistilBertModel.from_pretrained('distilbert-base-uncased')
 self.pre_classifier = nn.Linear(config.hidden_size, config.hidden_size)
 self.classifier = nn.Linear(config.hidden_size, config.num_labels)
 self.dropout = nn.Dropout(config.seq_classif_dropout)
 nn.init.xavier_normal_(self.classifier.weight)
 def forward(self, input_ids=None, attention_mask=None, head_mask=None, labels=None):
distilbert_output = self.distilbert(input_ids=input_ids,
attention_mask=attention_mask,
head_mask=head_mask)
 hidden_state = distilbert_output[0]
pooled_output = hidden_state[:, 0]
pooled_output = self.pre_classifier(pooled_output)
 pooled_output = nn.ReLU()(pooled_output)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 return logits
for idn in h_ids:
train.loc[(train['idhogar'] == idn), 'v18q1'] = train.query(f'idhogar == {idn}')['v18q'].sum()<feature_engineering>
max_seq_length = 256
tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
 class text_dataset(Dataset):
def __init__(self,x,y, transform=None):
self.x = x
self.y = y
self.transform = transform
def __getitem__(self,index):
tokenized_comment = tokenizer.tokenize(self.x[index])
 if len(tokenized_comment)> max_seq_length:
tokenized_comment = tokenized_comment[:max_seq_length]
ids_review = tokenizer.convert_tokens_to_ids(tokenized_comment)
 padding = [0] *(max_seq_length - len(ids_review))
ids_review += padding
assert len(ids_review)== max_seq_length
ids_review = torch.tensor(ids_review)
 hcc = self.y[index]
list_of_labels = [torch.from_numpy(hcc)]
return ids_review, list_of_labels[0]
def __len__(self):
return len(self.x)
for idn in h_ids:
test.loc[(test['idhogar'] == idn), 'v18q1'] = test.query(f'idhogar == {idn}')['v18q'].sum()<define_variables>
text_dataset(X_train, y_train ).__getitem__(6)[1]
cols = ['Id', 'idhogar', 'escolari', 'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4',
'estadocivil5', 'estadocivil6', 'estadocivil7', 'instlevel1', 'age', 'Target']
cols2 = cols[:-1]<feature_engineering>
batch_size = 32
training_dataset = text_dataset(X_train,y_train)
 test_dataset = text_dataset(X_test,y_test)
 dataloaders_dict = {'train': torch.utils.data.DataLoader(training_dataset, batch_size=batch_size, shuffle=False),
'val':torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 }
dataset_sizes = {'train':len(X_train),
'val':len(X_test)}
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = DistilBertForSequenceClassification(config)
 model.to(device)
 print(device)
train.loc[(train['rez_esc'].isnull())&(train['instlevel1'] == 1), 'rez_esc'] = 0
test.loc[(test['rez_esc'].isnull())&(test['instlevel1'] == 1), 'rez_esc'] = 0<filter>
def train_model(model, criterion, optimizer, scheduler, num_epochs=2):
model.train()
since = time.time()
print('starting')
 best_model_wts = copy.deepcopy(model.state_dict())
 best_loss = 100
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch+1, num_epochs))
print('-' * 10)
 for phase in ['train', 'val']:
if phase == 'train':
scheduler.step()
model.train()
else:
model.eval()
running_loss = 0.0
beta_score_accuracy = 0.0
micro_roc_auc_acc = 0.0
for inputs, hcc in dataloaders_dict[phase]:
inputs = inputs.to(device)
 hcc = hcc.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
 loss = criterion(outputs,hcc.float())
 if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
 micro_roc_auc_acc += accuracy_thresh(outputs.view(-1,6),hcc.view(-1,6))
epoch_loss = running_loss / dataset_sizes[phase]
epoch_micro_roc_acc = micro_roc_auc_acc / dataset_sizes[phase]
print('{} total loss: {:.4f} '.format(phase,epoch_loss))
print('{} micro_roc_auc_acc: {:.4f}'.format(phase, epoch_micro_roc_acc))
if phase == 'val' and epoch_loss < best_loss:
print('saving with loss of {}'.format(epoch_loss),
'improved over previous {}'.format(best_loss))
best_loss = epoch_loss
best_model_wts = copy.deepcopy(model.state_dict())
 torch.save(model.state_dict() , 'distilbert_model_weights.pth')
 print()
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
 time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(float(best_loss)))
 model.load_state_dict(best_model_wts)
 return model
print('done' )
a = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil2'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
b = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil3'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
c = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil4'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
d = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil5'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
e = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil6'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
f = train.loc[(~train['rez_esc'].isnull())&(train['estadocivil7'] == 1)&(train['escolari'] > 0)]['rez_esc'].mean()
a, b, c, d, e, f
train.loc[(train['rez_esc'].isnull())&(train['estadocivil2'] == 1), 'rez_esc'] = 3
train.loc[(train['rez_esc'].isnull())&(train['estadocivil3'] == 1), 'rez_esc'] = 0
train.loc[(train['rez_esc'].isnull())&(train['estadocivil4'] == 1), 'rez_esc'] = 0
train.loc[(train['rez_esc'].isnull())&(train['estadocivil5'] == 1), 'rez_esc'] = 2
train.loc[(train['rez_esc'].isnull())&(train['estadocivil6'] == 1), 'rez_esc'] = 0
train.loc[(train['rez_esc'].isnull())&(train['estadocivil7'] == 1), 'rez_esc'] = 1
test.loc[(test['rez_esc'].isnull())&(test['estadocivil2'] == 1), 'rez_esc'] = 3
test.loc[(test['rez_esc'].isnull())&(test['estadocivil3'] == 1), 'rez_esc'] = 0
test.loc[(test['rez_esc'].isnull())&(test['estadocivil4'] == 1), 'rez_esc'] = 0
test.loc[(test['rez_esc'].isnull())&(test['estadocivil5'] == 1), 'rez_esc'] = 2
test.loc[(test['rez_esc'].isnull())&(test['estadocivil6'] == 1), 'rez_esc'] = 0
test.loc[(test['rez_esc'].isnull())&(test['estadocivil7'] == 1), 'rez_esc'] = 1<count_missing_values>
lrlast =.001
lrmain = 3e-5
optim1 = torch.optim.Adam(model.parameters() ,lrmain)
 optimizer_ft = optim1
criterion = nn.BCEWithLogitsLoss()
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=3, gamma=0.1 )
train['rez_esc'].isnull().sum() , test['rez_esc'].isnull().sum()<define_variables>
model_ft1 = train_model(model, criterion, optimizer_ft, exp_lr_scheduler,num_epochs=8 )
cols = ['Id', 'idhogar', 'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5',
'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']<filter>
x_test = test['comment_text']
y_test = np.zeros(x_test.shape[0]*6 ).reshape(x_test.shape[0],6)
 test_dataset = text_dataset(x_test,y_test)
 prediction_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False)
 def preds(model,test_loader):
predictions = []
for inputs, sentiment in test_loader:
inputs = inputs.to(device)
 sentiment = sentiment.to(device)
 with torch.no_grad() :
outputs = model(inputs)
 outputs = torch.sigmoid(outputs)
 predictions.append(outputs.cpu().detach().numpy().tolist())
 return predictions
print(train.loc[(train['idhogar'] == 326), 'meaneduc'].values)
 print(train.loc[(train['idhogar'] == 1959), 'meaneduc'].values)
 print(train.loc[(train['idhogar'] == 2908), 'meaneduc'].values )<prepare_x_and_y>
predictions = preds(model=model_ft1,test_loader=prediction_dataloader)
 predictions = np.array(predictions)[:,0]
def meaneduc_correction(null_view, df, hids):

 for idn in hids:
a = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel1'] == 1)].shape[0]
b = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel2'] == 1)].shape[0]
c = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel3'] == 1)].shape[0]
d = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel4'] == 1)].shape[0]
e = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel5'] == 1)].shape[0]
f = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel6'] == 1)].shape[0]
g = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel7'] == 1)].shape[0]
h = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel8'] == 1)].shape[0]
i = null_view.loc[(null_view['idhogar']==idn)&(null_view['instlevel9'] == 1)].shape[0]
mean_educ =(a*0 + b*4 + c*8 + d*2 + e*4 + f + g*2 + h*4 + i)/(a+b+c+d+e+f+g+h+i)
 df.loc[(df['meaneduc'].isnull())&(df['idhogar'] == idn), 'meaneduc'] = mean_educ
df.loc[(df['SQBmeaned'].isnull())&(df['idhogar'] == idn), 'SQBmeaned'] = mean_educ**2<compute_test_metric>
submission = pd.DataFrame(predictions,columns=['toxic','severe_toxic','obscene','threat','insult','identity_hate'])
 test[['toxic','severe_toxic','obscene','threat','insult','identity_hate']]=submission
final_sub = test[['id','toxic','severe_toxic','obscene','threat','insult','identity_hate']]
final_sub.head()
<filter><EOS> 
final_sub.to_csv('submissions.csv',index=False)
 final_sub.head()