Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
X = train_data.drop(['PassengerId', 'Name', 'Ticket', 'Cabin', 'Embarked'], axis=1)
 X_test = test_data.drop(['PassengerId','Name', 'Ticket', 'Cabin', 'Embarked'], axis=1 )<categorify>
warnings.filterwarnings(action='once')
 device = torch.device('cuda' )
X = pd.get_dummies(X)
 X_test = pd.get_dummies(X_test)
 X.fillna(X.mean() ,inplace=True)
 X_test.fillna(X_test.mean() ,inplace=True )<count_missing_values>
import pdb
X.isnull().sum()<count_missing_values>
MODELS_PATH = '.. /input/submission-toxicity-classification'
X_test.isnull().sum()<prepare_x_and_y>
BERT_MODEL_PATH = '.. /input/submission-toxicity-classification/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
features = ["Pclass", "Sex_female", "Age", "Fare", "SibSp", "Parch"]
y= X['Survived']
X = pd.DataFrame(X, columns = features)
 X_test = pd.DataFrame(X_test, columns = features)
 for col in features:
X[col] =(X[col] - X[col].mean())/ X[col].std()
X_test[col] =(X_test[col] - X_test[col].mean())/ X_test[col].std()
for col in features:
X[col] =(X[col] - X[col].min())/(X[col].max() - X[col].min())
 X_test[col] =(X_test[col] - X_test[col].min())/(X_test[col].max() - X_test[col].min())
<save_to_csv>
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv")
 test_df['comment_text'] = test_df['comment_text'].astype(str )
model = RandomForestClassifier(n_estimators = 100, max_features='auto', criterion='entropy',max_depth=10)
 model.fit(X, y)
 predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('random_forest_submission.csv', index=False)
 print("Your submission was successfully saved!")
 cv_scores = cross_val_score(model, X, y,
cv=5,
scoring='accuracy')
 print("Cross-validation accuracy: %f" % cv_scores.mean() )<load_from_csv>
def bert_get_preds(model, x_test):
test_preds = np.zeros(( len(x_test)))
 test = torch.utils.data.TensorDataset(torch.tensor(x_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)
 tk0 = tqdm(test_loader)
 for i,(x_batch,)in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 test_preds[i*32:(i+1)*32] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()
return test_pred
df = pd.read_csv("random_forest_submission.csv")
 df<data_type_conversions>
def gpt2_get_preds(model, x_test):
test_preds = np.zeros(( len(x_test)))
 test = torch.utils.data.TensorDataset(torch.tensor(x_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)
 tk0 = tqdm(test_loader)
 for i,(x_batch,)in enumerate(tk0):
pred = model(x_batch.to(device))
test_preds[i*32:(i+1)*32] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()
return test_pred
X = X.to_numpy()
y = y.to_numpy().reshape(-1, 1)
 X_test = X_test.to_numpy()<define_variables>
def bert_convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"])+[0] *(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens )
def batch_data(batch_size, input_data, target, test_data, train_type = "regression", val_size=0.1):

 if train_type == "regression":
target_tensor = torch.FloatTensor(target)
 elif train_type == "classification":
target_tensor = torch.LongTensor(target)
 target_tensor = target_tensor.squeeze()
input_tensor = torch.FloatTensor(input_data)
 test_tensor = torch.FloatTensor(test_data)
 train_dataset = TensorDataset(input_tensor, target_tensor)
 num_train = len(train_dataset)
 indicies = list(range(num_train))
np.random.shuffle(indicies)
 val_split = int(np.floor(val_size * num_train))
train_idx, val_idx = indicies[val_split:], indicies[:val_split]
train_sampler = SubsetRandomSampler(train_idx)
 val_sampler = SubsetRandomSampler(val_idx)
 train_loader = DataLoader(train_dataset, batch_size=batch_size, sampler = train_sampler)
 val_loader = DataLoader(train_dataset, batch_size=batch_size, sampler = val_sampler)
 test_loader = DataLoader(test_tensor, batch_size=len(test_data))
return train_loader, val_loader, test_loader 
<define_variables>
def gpt2_convert_lines(example, max_seq_length, tokenizer):
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)> max_seq_length:
tokens_a = tokens_a[-max_seq_length:]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(tokens_a)+ [0]*(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens )
batch_size = 32
train_loader, val_loader, test_loader = batch_data(batch_size, X, y, X_test )<normalization>
class BertHead(nn.Module):
def __init__(self, config, hidden_units=256, num_aux_targets=6):
super(BertHead, self ).__init__()
self.hidden_units = hidden_units
self.num_aux_targets = num_aux_targets
self.config = config
self.bert_model = BertForSequenceClassification.from_pretrained(config, cache_dir=None, num_labels=hidden_units)
 param_optimizer = list(self.bert_model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
self.fc_out = nn.Linear(768, 1)
 self.fc_aux_out = nn.Linear(768, num_aux_targets)
 self.fc_dp = nn.Dropout(p=0.4)
 self.optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
{'params': [p for p in self.fc_out.parameters() ], 'weight_decay': 0.0},
{'params': [p for p in self.fc_aux_out.parameters() ], 'weight_decay': 0.0}
]
def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None):
_, pooled_output = self.bert_model.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False)
 pooled_output = self.bert_model.dropout(pooled_output)
 logits = self.fc_dp(pooled_output)
 out = self.fc_out(logits)
 out_aux = self.fc_aux_out(logits)
 return torch.cat([out, out_aux], 1 )
class LinearRegression(nn.Module):
def __init__(self):
super().__init__()
self.fc1 = nn.Linear(6, 20)
 self.fc2 = nn.Linear(20, 1)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
x = self.fc1(x)
 x = self.sigmoid(x)
 x = self.fc2(x)
 return x 
<load_pretrained>
bert_tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None, do_lower_case=True )
def init_weights(m):
if type(m)== nn.Linear:
torch.nn.init.xavier_uniform(m.weight)
 m.bias.data.fill_(0.01 )<choose_model_class>
bert_base_models = ['bert_epoch_2_lb_093967.bin', 'bert_epoch_2_lb_093916.bin', 'bert_epoch_1_lb_093970.bin']
linear_regression_model = LinearRegression()
linear_regression_model.apply(init_weights)
 print(linear_regression_model )<choose_model_class>
x_test = bert_convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), 220, bert_tokenizer )
lr = 0.01
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(linear_regression_model.parameters() , lr=lr, momentum=0.9)
 batch_size = 32<train_model>
bert_base_preds = np.zeros(( x_test.shape[0], len(bert_base_models)))
 for i, bert_model in enumerate(bert_base_models):
model = BertHead('.. /input/bert_uncased')
 model.load_state_dict(torch.load(os.path.join(MODELS_PATH, bert_model)))
 model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
preds = bert_get_preds(model, x_test)
 bert_base_preds[:,i] = preds
del model
gc.collect()
def train_model(model, batch_size, epochs, cost_function, print_every = 100):
val_loss_min = np.Inf
for e in range(epochs):
val_loss = 0.0
train_loss = 0.0
model.train()
for inputs, labels in train_loader:
optimizer.zero_grad()
output = model(inputs)
 loss = cost_function(output, labels)
 loss.backward()
optimizer.step()
train_loss += loss.item() * batch_size
model.eval()
for inputs, labels in val_loader:
output = model(inputs)
 loss = cost_function(output, labels)
 val_loss += loss.item() * batch_size
train_loss = train_loss / len(train_loader.sampler)
 val_loss = val_loss / len(val_loader.sampler)
 if val_loss <= val_loss_min:
print('Validation loss decreased({:.6f} --> {:.6f} ).Saving model...'.format(
 val_loss_min, val_loss))
val_loss_min = val_loss
torch.save(model.state_dict() , 'model_linear.pt')
 if epochs % print_every == 0:
print('Epoch: {} \tTraining Loss: {:.6f} \tValidation Loss: {:.6f}'.format(e, train_loss, val_loss))
print("Best model with validation loss: {}".format(val_loss_min)) 
<train_model>
class GPT2ClassificationHeadModel(GPT2PreTrainedModel):
def __init__(self, config, clf_dropout=0.4, num_aux_targets=6):
super(GPT2ClassificationHeadModel, self ).__init__(config)
 self.transformer = GPT2Model(config)
 self.dropout = nn.Dropout(clf_dropout)
 self.fc_out = nn.Linear(config.n_embd * 2, 1)
 self.fc_aux_out = nn.Linear(config.n_embd * 2, num_aux_targets)
 def forward(self, input_ids, position_ids=None, token_type_ids=None, lm_labels=None, past=None):
hidden_states, presents = self.transformer(input_ids, position_ids, token_type_ids, past)
 avg_pool = torch.mean(hidden_states, 1)
 max_pool, _ = torch.max(hidden_states, 1)
 logits = torch.cat(( avg_pool, max_pool), 1)
 logits = self.dropout(logits)
 out = self.fc_out(logits)
 out_aux = self.fc_aux_out(logits)
 return torch.cat([out, out_aux], 1 )
train_model(linear_regression_model, batch_size, epochs=3000, cost_function=criterion )<load_from_disk>
gpt2_tokenizer = GPT2Tokenizer.from_pretrained('.. /input/submission-toxicity-classification/gpt2-models/' )
linear_regression_model.load_state_dict(torch.load('model_linear.pt'))<find_best_params>
gpt2_base_models = ['gpt2_epoch_1_lb_093912.bin', 'gpt2_epoch_1_lb_093904.bin', 'gpt2_k_epoch_2_lb_093902.bin']
with torch.no_grad() :
for data in test_loader:
output = linear_regression_model(data)
 preds = torch.round(output)
 preds = preds.squeeze()
survived = preds.numpy() 
<data_type_conversions>
x_test = gpt2_convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), 220, gpt2_tokenizer )
survived = survived.astype('int' )<save_to_csv>
gpt2_base_preds = np.zeros(( x_test.shape[0], len(gpt2_base_models)))
 for i, gpt2_model in enumerate(gpt2_base_models):
model = GPT2ClassificationHeadModel.from_pretrained('.. /input/submission-toxicity-classification/gpt2-models/')
 model.load_state_dict(torch.load(os.path.join(MODELS_PATH, gpt2_model)))
 model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
preds = gpt2_get_preds(model, x_test)
 gpt2_base_preds[:,i] = preds
del model
gc.collect()
submission = pd.DataFrame({'PassengerId': test_data['PassengerId'], 'Survived': survived})
 submission.to_csv('submission_regression.csv', index=False)
 print("Your submission was successfully saved!" )<load_from_csv>
test_pred =(np.average(bert_base_preds, axis=1)+ np.average(gpt2_base_preds, axis=1)) / 2
df = pd.read_csv("submission_regression.csv")
 df<define_variables>
submission = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': test_pred
})
 submission.to_csv('submission.csv', index=False )