Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<prepare_x_and_y>
import pandas as pd
import numpy as np
import nltk
import re
from keras.preprocessing import text, sequence
from sklearn.model_selection import train_test_split
import torch
from torch import nn
from torch.utils import data
from torch.nn import functional as F
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
import time
import random
import gc
import os
import multiprocessing as mp
import matplotlib.pyplot as plt
train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test<concatenate>
def load_data(size=4):
start = time.time()
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 train = clean_text(train)
 test = clean_text(test)
 x_train = train['comment_text']
y_train = train['target']
x_test = test['comment_text']
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
print('Data was successfully read in and preprocessed.Time taken: ',
round(( time.time() - start)/60, 2), ' minutes')
 return x_train, x_test, y_train, y_aux_train, test
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 print(features.shape )<data_type_conversions>
def tokenize_text(x_train, x_test, pad_len):
MAX_WORDS = 100_000
start = time.time()
tokenizer = text.Tokenizer(num_words = MAX_WORDS)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_train = sequence.pad_sequences(x_train, maxlen=pad_len)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_test = sequence.pad_sequences(x_test, maxlen=pad_len)
 print('Text has been tokenized in', round(( time.time() - start)/60, 2), ' minutes')
 return x_train, x_test, tokenizer
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str)
 features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))<sort_values>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path, encoding="utf8")as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
def make_embedder(tokenizer):
GLOVE_PATH = '.. /input/crawl300d2m/crawl-300d-2M.vec'
CRAWL_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
start = time.time()
glove_matrix, unknown_words = build_matrix(tokenizer.word_index, GLOVE_PATH)
 crawl_matrix, unknown_words = build_matrix(tokenizer.word_index, CRAWL_PATH)
 embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis = -1)
 gc.collect()
print('Embedding matrix was successfully made.Time taken:',
round(( time.time() - start)/60,2), ' minutes')
 return embedding_matrix
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics.append(i)
 features.update(features[numerics].fillna(0))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics2.append(i)
 skew_features = features[numerics2].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
<categorify>
def make_train_test(x_train, x_test, y_train, y_aux_train):
x_train_torch = torch.tensor(x_train, dtype=torch.long ).cuda()
x_test_torch = torch.tensor(x_test, dtype=torch.long ).cuda()
y_train_torch = torch.tensor(
 np.hstack([y_train[:, np.newaxis], y_aux_train]),
dtype=torch.float32 ).cuda()
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 test_dataset = data.TensorDataset(x_test_torch)
 print('Pytorch data has been made.')
 return train_dataset, test_dataset, y_train_torch
features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 print(features.shape)
 final_features = pd.get_dummies(features ).reset_index(drop=True)
 print(final_features.shape)
 X = final_features.iloc[:len(y), :]
X_sub = final_features.iloc[len(X):, :]
print('X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape )<train_model>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets, LSTM_UNITS, max_features):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers])
 overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X = X.drop(overfit, axis=1 ).copy()
X_sub = X_sub.drop(overfit, axis=1 ).copy()
print('X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape)
<compute_train_metric>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def train_model(model, train, test, loss_fn, output_dim,
lr=0.001, batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
param_lrs = [{'params': param, 'lr': lr} for param in model.parameters() ]
optimizer = torch.optim.Adam(param_lrs, lr=lr)
 scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda epoch: 0.6 ** epoch)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
for epoch in range(n_epochs):
start_time = time.time()
scheduler.step()
model.train()
avg_loss = 0.
 for data in tqdm(train_loader, disable=False):
x_batch = data[:-1]
y_batch = data[-1]
y_pred = model(*x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
y_pred = sigmoid(model(*x_batch ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, elapsed_time))
if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y,
scoring="neg_mean_squared_error",
cv=kfolds))
return(rmse)
 alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7, alphas=alphas2,
random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() ,
SVR(C= 20, epsilon= 0.008, gamma=0.0003,))
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =42)
 lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 xgboost = XGBRegressor(learning_rate=0.01, n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet,
gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True)
 print('TEST score on CV')
 score = cv_rmse(ridge)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("Lightgbm score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(gbr)
 print("GradientBoosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )<train_model>
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 seed_everything()
stack_gen_model = stack_gen.fit(np.array(X), np.array(y))
print(datetime.now() , 'elasticnet')
 elastic_model_full_data = elasticnet.fit(X, y)
 print(datetime.now() , 'lasso')
 lasso_model_full_data = lasso.fit(X, y)
 print(datetime.now() , 'ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print(datetime.now() , 'svr')
 svr_model_full_data = svr.fit(X, y)
 print(datetime.now() , 'GradientBoosting')
 gbr_model_full_data = gbr.fit(X, y)
 print(datetime.now() , 'xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print(datetime.now() , 'lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y )<predict_on_test>
x_train, x_test, y_train, y_aux_train, test = load_data(size=1)
 x_train, x_test, tokenizer = tokenize_text(x_train, x_test, pad_len = 205)
 embedding_matrix = make_embedder(tokenizer )
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.1 * lasso_model_full_data.predict(X)) + \ 
(0.1 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.15 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.25 * stack_gen_model.predict(np.array(X))))
print('RMSLE score on train data:')
 print(rmsle(y, blend_models_predict(X)) )<load_from_csv>
lstm_units = 128
num_epochs = 5
pad_length = 250
x_train, x_test, y_train, y_aux_train, test = load_data(size=1)
 x_train, x_test, tokenizer = tokenize_text(x_train, x_test, pad_len = pad_length)
 embedding_matrix = make_embedder(tokenizer)
 train_dataset, test_dataset, y_train_torch = make_train_test(x_train, x_test, y_train, y_aux_train)
 all_test_preds = []
start = time.time()
NUM_MODELS = 2
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(0 + model_idx)
 model = NeuralNet(
 embedding_matrix, y_aux_train.shape[-1],
lstm_units, len(tokenizer.word_index)+ 1)
 model.cuda()
test_preds = train_model(
 model, train_dataset, test_dataset,
output_dim=y_train_torch.shape[-1], n_epochs = num_epochs,
loss_fn=nn.BCEWithLogitsLoss(reduction='mean'))
all_test_preds.append(test_preds)
 print('Total time taken: ', round(( time.time() - start)/60,2), 'minutes')
 print()
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_sub)))
 print('Blend with Top Kernals submissions', datetime.now() ,)
 sub_1 = pd.read_csv('.. /input/top-10-0-10943-stacking-mice-and-brutal-force/House_Prices_submit.csv')
 sub_2 = pd.read_csv('.. /input/hybrid-svm-benchmark-approach-0-11180-lb-top-2/hybrid_solution.csv')
 sub_3 = pd.read_csv('.. /input/lasso-model-for-regression-problem/lasso_sol22_Median.csv')
 submission.iloc[:,1] = np.floor(( 0.25 * np.floor(np.expm1(blend_models_predict(X_sub)))) + 
(0.25 * sub_1.iloc[:,1])+ 
(0.25 * sub_2.iloc[:,1])+ 
(0.25 * sub_3.iloc[:,1]))<save_to_csv>
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )